diff --git a/gem5.config.json b/gem5.config.json index dab2bc4..184ad4e 100644 --- a/gem5.config.json +++ b/gem5.config.json @@ -5,7 +5,7 @@ "dataSource": "gem5-vision", "database": "gem5-vision", "collection": "resources", - "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "url": "http://localhost:7071/api/resources", "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", "isMongo": true diff --git a/pages/api/mongodb/getFilters.js b/pages/api/mongodb/getFilters.js index 4baa130..f92f2be 100644 --- a/pages/api/mongodb/getFilters.js +++ b/pages/api/mongodb/getFilters.js @@ -3,60 +3,47 @@ import getToken from "./getToken"; /** * @function getFilters * @async - * @description This asynchronous function fetches distinct categories, architectures, and gem5 versions from a specified data source using MongoDB aggregation pipeline. + * @description This asynchronous function fetches distinct categories, architectures, and gem5 versions from MongoDB database * It takes in an access token, URL, data source, database, and collection as input parameters, and returns an object containing the * distinct categories, architectures, and gem5 versions sorted in ascending or descending order. * @param {string} accessToken - The access token for authentication. * @param {string} url - The URL of the data source. - * @param {string} dataSource - The name of the data source. - * @param {string} database - The name of the database. - * @param {string} collection - The name of the collection. * @returns {Object} - An object containing the distinct categories, architectures, and gem5 versions sorted in ascending or descending order. */ -async function getFilters(accessToken, url, dataSource, database, collection) { +async function getFilters(accessToken, url) { // get all distinct categories from resources - const res = await fetch(`${url}/action/aggregate`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Request-Headers': '*', - "Authorization": "Bearer " + accessToken, - }, - body: JSON.stringify({ - "dataSource": dataSource, - "database": database, - "collection": collection, - "pipeline": [ - { - "$unwind": "$gem5_versions" - }, - { - "$group": { - "_id": null, - "category": { "$addToSet": "$category" }, - "architecture": { "$addToSet": "$architecture" }, - "gem5_versions": { "$addToSet": "$gem5_versions" } - } - } - ] - }) - }).catch(err => console.log(err)); - let filters = await res.json(); - if (res.status != 200 || filters['documents'].length == 0) { + try { + const res = await fetch(`${url}/filters`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + // 'x-functions-key': accessToken, // Using API token as x-functions-key for Azure Functions + }, + }); + + // Check if status is not 200 + if (res.status !== 200) { + console.log("Error: " + res.status); + return { + "category": [], + "architecture": [], + "gem5_versions": [] + }; + } + + // Parse the response + const filters = await res.json(); + + // Return the filters directly since the API already formats them correctly + return filters; + } catch (err) { + console.log("Error fetching filters:", err); return { "category": [], "architecture": [], "gem5_versions": [] - } + }; } - filters['documents'][0]['architecture'] = filters['documents'][0]['architecture'].filter(architecture => architecture != null); - delete filters['documents'][0]['_id']; - - filters['documents'][0]['gem5_versions'] = filters['documents'][0]['gem5_versions'] - filters['documents'][0]['category'].sort(); - filters['documents'][0]['architecture'].sort(); - filters['documents'][0]['gem5_versions'].sort().reverse(); - return filters['documents'][0]; } /** @@ -69,6 +56,6 @@ export default async function getFiltersMongoDB(database) { let privateResources = process.env.SOURCES; let privateResource = privateResources[database]; let privateAccessToken = await getToken(database); - let privateFilters = await getFilters(privateAccessToken, privateResource.url, privateResource.dataSource, privateResource.database, privateResource.collection); + let privateFilters = await getFilters(privateAccessToken, privateResource.url); return privateFilters; } diff --git a/pages/api/mongodb/getResourceByID.js b/pages/api/mongodb/getResourceByID.js index 0fb515e..d68345f 100644 --- a/pages/api/mongodb/getResourceByID.js +++ b/pages/api/mongodb/getResourceByID.js @@ -16,77 +16,37 @@ import getToken from "./getToken"; * If not provided, the latest version will be retrieved. * @returns {Object} - The retrieved resource object, including its metadata and associated workloads. */ -async function getResourceByID(token, url, dataSource, database, collection, id, version = null) { - const res = await fetch(`${url}/action/find`, { - method: 'POST', +async function getResourceByID(token, url, id, version = null) { + + let params = new URLSearchParams(); + params.append("id", id) + if (version == null) { + params.append("resource_version", version) + } + const res = await fetch(`${url}/find-resource-by-id?${params.toString()}`, { + method: 'GET', headers: { 'Content-Type': 'application/json', // 'api-key': 'pKkhRJGJaQ3NdJyDt69u4GPGQTDUIhHlx4a3lrKUNx2hxuc8uba8NrP3IVRvlzlo', - 'Access-Control-Request-Headers': '*', // 'origin': 'https://gem5vision.github.io', - "Authorization": "Bearer " + token, }, - body: JSON.stringify({ - "dataSource": dataSource, - "database": database, - "collection": collection, - "filter": version ? { - "id": id, - "resource_version": version - } : { - "id": id - } - }) }).catch(err => console.log(err)); let resource = await res.json(); - if (res.status != 200 || resource['documents'] === null || resource['documents'].length === 0) { + if (res.status != 200 || resource === null || resource.length === 0) { return { error: 'Resource not found' } } - resource = resource['documents'].sort((a, b) => -compareVersions(a.resource_version, b.resource_version))[0]; + resource = resource.sort((a, b) => -compareVersions(a.resource_version, b.resource_version))[0]; - const dependendWorkloads = await fetch(`${url}/action/aggregate`, { - method: 'POST', + const dependendWorkloads = await fetch(`${url}/get-dependent-workloads?${params.toString()}`, { + method: 'GET', headers: { 'Content-Type': 'application/json', - 'Access-Control-Request-Headers': '*', - 'Authorization': 'Bearer ' + token, - }, - body: JSON.stringify({ - "dataSource": dataSource, - "database": database, - "collection": collection, - "pipeline": [ - { - "$match": { - "category": "workload" - } - }, - { - "$addFields": { - "resources": { - "$objectToArray": "$resources" - } - } - }, - { - "$unwind": "$resources" - }, - { - "$match": { - "resources.v": id - } - }, - { - "$group": { - "_id": "$id", - } - } - ] - }) + } }).catch(err => console.log(err)); let workloads = await dependendWorkloads.json(); - resource.workloads_mapping = Object.values(workloads['documents']).map(workload => workload['_id']); + console.log(workloads) + resource.workloads_mapping = Object.values(workloads).map(workload => workload['_id']); return resource; } @@ -101,7 +61,7 @@ async function getResourceByID(token, url, dataSource, database, collection, id, export default async function getResourceByIDMongoDB(id, database = null, version = null) { const token = await getToken(database); let privateResources = process.env.SOURCES[database]; - const resource = await getResourceByID(token, privateResources.url, privateResources.dataSource, privateResources.database, privateResources.collection, id, version); + const resource = await getResourceByID(token, privateResources.url,id, version); resource['database'] = database; return resource; } \ No newline at end of file diff --git a/pages/api/mongodb/getResourcesByQuery.js b/pages/api/mongodb/getResourcesByQuery.js index 4045658..58c07e8 100644 --- a/pages/api/mongodb/getResourcesByQuery.js +++ b/pages/api/mongodb/getResourcesByQuery.js @@ -1,330 +1,5 @@ import getToken from "./getToken"; -/** - * @function getSort - * @description This function returns a sort object based on the provided sort parameter. - * The returned sort object can be used in a MongoDB query to specify the sorting order of the results. - * @param {string} sort - The sort parameter to determine the sorting order. - * Possible values are: "date" to sort by date in descending order, - * "name" to sort by name (or ID) in ascending order, - * "version" to sort by version in descending order, - * "id_asc" to sort by ID in ascending order, - * "id_desc" to sort by ID in descending order. - * If not provided or an invalid value is provided, resources will be sorted by score in descending order by default. - * @returns {Object} - A sort object that can be used in a MongoDB query to specify the sorting order of the results. - * @property {number} date - If "date" is provided as the sort parameter, - * the value of this property will be -1 to sort by date in descending order. - * @property {number} id - If "name", "id_asc", or "id_desc" is provided as the sort parameter, - * the value of this property will be either 1 or -1 to specify the sorting order of the ID field. - * @property {number} ver_latest - If "version" is provided as the sort parameter, - * the value of this property will be -1 to sort by version in descending order. - * @property {number} score - If an invalid or no sort parameter is provided, - * the value of this property will be -1 to sort by score in descending order by default. - */ -function getSort(sort) { - switch (sort) { - case "date": - return { date: -1 }; - case "name": - return { id: 1 }; - case "version": - return { ver_latest: -1 }; - case "id_asc": - return { id: 1 }; - case "id_desc": - return { id: -1 }; - default: - return { - score: -1, - }; - } -} - -/** - * @function getLatestVersionPipeline - * @description This function returns a MongoDB aggregation pipeline that can be used - * to retrieve documents with the latest version based on the "resource_version" field. - * The pipeline performs several stages to split the "resource_version" into parts, convert them to integers, - * sort them in descending order, group them by "id" field, and return the document with the latest version along - * with its "id" and "latest_version" fields. - * @returns {Array} - An array representing the MongoDB aggregation pipeline to retrieve documents with the latest version. - */ -function getLatestVersionPipeline() { - let pipeline = [ - { - $addFields: { - resource_version_parts: { - $map: { - input: { - $split: ["$resource_version", "."], - }, - in: { $toInt: "$$this" }, - }, - }, - }, - }, - { - $sort: { - id: 1, - "resource_version_parts.0": -1, - "resource_version_parts.1": -1, - "resource_version_parts.2": -1, - "resource_version_parts.3": -1, - }, - }, - { - $group: { - _id: "$id", - latest_version: { - $first: "$resource_version", - }, - document: { $first: "$$ROOT" }, - }, - }, - { - $replaceRoot: { - newRoot: { - $mergeObjects: [ - "$document", - { - id: "$_id", - latest_version: "$latest_version", - }, - ], - }, - }, - }, - ] - return pipeline; -} - -/** - * @function getSearchPipeline - * @description This function returns a MongoDB aggregation pipeline that can be used to perform text search - * on a collection based on the provided query object. - * The pipeline uses the $search stage to perform compound text search on multiple fields, - * including "id", "description", "category", "architecture", and "tags". - * It applies boost to the "id" field to prioritize matching results, - * and also adds a "score" field to each document to represent the search score. - * @param {Object} queryObject - The query object containing the search query and options. - * @param {string} queryObject.query - The search query string. - * @returns {Array} - An array representing the MongoDB aggregation pipeline to perform text search. - */ -function getSearchPipeline(queryObject) { - // getting current gem5_version to boost search for resources compatible with latest gem5 release - const gem5_version = process.env.GEM5_VERSION; - - let pipeline = [ - { - $search: { - compound: { - should: [ - { - text: { - path: "id", - query: queryObject.query, - score: { - boost: { - value: 10, - }, - }, - }, - }, - { - text: { - path: "gem5_versions", - query: gem5_version, - score: { - boost: { - value: 10, - }, - }, - }, - } - ], - must: [ - { - text: { - query: queryObject.query, - path: [ - "id", - "desciption", - "category", - "architecture", - "tags", - ], - fuzzy: { - maxEdits: 2, - maxExpansions: 100, - }, - }, - }, - ], - }, - }, - }, - { - $addFields: { - score: { - $meta: "searchScore", - }, - }, - }, - ]; - return pipeline; -} - -/** - * @function getFilterPipeline - * @description This function returns a MongoDB aggregation pipeline that can be used to apply filters - * to a collection based on the provided query object. - * The pipeline applies filters based on tags, gem5_versions, category, and architecture fields in the documents. - * It unwinds and matches the specified fields in the query object, and groups the results to eliminate duplicates. - * The resulting pipeline can be used in conjunction with other aggregation stages to further process the filtered documents. - * @param {Object} queryObject - The query object containing the filter criteria. - * @param {Array} queryObject.tags - An array of tags to filter by. - * @param {Array} queryObject.gem5_versions - An array of gem5 versions to filter by. - * @param {Array} queryObject.category - An array of categories to filter by. - * @param {Array} queryObject.architecture - An array of architectures to filter by. - * @returns {Array} - An array representing the MongoDB aggregation pipeline to apply filters. - */ -function getFilterPipeline(queryObject) { - let pipeline = [] - // adding filter by gem5 version if the user has selected one - if (queryObject.tags) { - pipeline.push(...[ - { - $addFields: { - tag: "$tags", - }, - }, - { - $unwind: "$tag", - }, - { - $match: { - tag: { - $in: queryObject.tags || [], - }, - }, - }, - { - $group: { - _id: "$_id", - doc: { - $first: "$$ROOT", - }, - }, - }, - { - $replaceRoot: { - newRoot: "$doc", - }, - }, - ]); - } - if (queryObject.gem5_versions) { - pipeline.push(...[ - { - $addFields: { - version: "$gem5_versions", - }, - }, - { - $unwind: "$version", - }, - { - $match: { - version: { - $in: queryObject.gem5_versions || [], - }, - }, - }, - { - $group: { - _id: "$_id", - doc: { - $first: "$$ROOT", - }, - }, - }, - { - $replaceRoot: { - newRoot: "$doc", - }, - }, - ]); - } - // adding the other fileters such as category and architecture to the serch pipeline - let match = []; - if (queryObject.category) { - match.push({ category: { $in: queryObject.category || [] } }); - } - if (queryObject.architecture) { - match.push({ architecture: { $in: queryObject.architecture || [] } }); - } - // adding the match to the pipeline if there are some filters selected - if (match.length > 0) { - pipeline.push({ - $match: { - $and: match, - }, - }); - } - return pipeline; -} - -/** - * @function getSortPipeline - * @description This function returns a MongoDB aggregation pipeline that can be used to apply sorting to a collection based on the provided query object. - * The pipeline adds a field "ver_latest" to each document in the collection, representing the maximum value in the "gem5_versions" array field. - * It then sorts the documents based on the specified sort criteria from the query object using the "getSort" helper function. - * @param {Object} queryObject - The query object containing the sort criteria. - * @param {string} queryObject.sort - The sort criteria to apply. - * Should be a string in the format "field:direction", where "field" is the field to sort by and "direction" - * is the sorting direction ("asc" for ascending, "desc" for descending). - * @returns {Array} - An array representing the MongoDB aggregation pipeline to apply sorting. - */ -function getSortPipeline(queryObject) { - let pipeline = [ - { - $addFields: { - ver_latest: { - $max: "$gem5_versions", - }, - }, - }, - { - $sort: getSort(queryObject.sort), - }]; - return pipeline; -} - -/** - * @function getPagePipeline - * @description This function returns a MongoDB aggregation pipeline that can be used to implement pagination for a collection, - * based on the provided current page number and page size. - * The pipeline uses the MongoDB $setWindowFields, $skip, and $limit stages to calculate the total count of documents in the collection, - * skip the appropriate number of documents based on the current page number and page size, and limit the number of documents returned per page. - * @param {number} currentPage - The current page number. - * @param {number} pageSize - The number of documents to display per page. - * @returns {Array} - An array representing the MongoDB aggregation pipeline to implement pagination. - */ -function getPagePipeline(currentPage, pageSize) { - let pipeline = [ - { - $setWindowFields: { output: { totalCount: { $count: {} } } }, - }, - { - $skip: (currentPage - 1) * pageSize, - }, - { - $limit: pageSize, - }, - ]; - return pipeline; -} - /** * @function getPipeline * @description This function returns a MongoDB aggregation pipeline based on the provided query object, @@ -339,25 +14,66 @@ function getPagePipeline(currentPage, pageSize) { */ function getPipeline(queryObject, currentPage, pageSize) { - let pipeline = []; + // Initialize URL parameters + let params = new URLSearchParams(); + if (queryObject.query.trim() === "") { if (queryObject.sort === "relevance") { queryObject.sort = "default"; } } - // adding search query if something is entered - if (queryObject.query.trim() !== "") { - pipeline.push(...getSearchPipeline(queryObject)); + // Add search term (contains-str) + if (queryObject.query && queryObject.query.trim() !== "") { + params.append('contains-str', queryObject.query.trim()); + } else { + // Default empty search term + params.append('contains-str', ''); } - // adding the filters to the pipeline - pipeline.push(...getFilterPipeline(queryObject)); - // getting latest resource version for each resource and removing the rest - pipeline.push(...getLatestVersionPipeline()); - // adding the sorting to the pipeline - pipeline.push(...getSortPipeline(queryObject)); - // adding the pagination to the pipeline - pipeline.push(...getPagePipeline(currentPage, pageSize)); - return pipeline; + + // Add filter criteria (must-include) + const filterParams = buildMustIncludeParams(queryObject); + if (filterParams) { + params.append('must-include', filterParams); + } + + // Add pagination parameters + params.append('page', currentPage); + params.append('page-size', pageSize); + params.append('sort', queryObject.sort); + + return params; +} +/** + * @function buildMustIncludeParams + * @description Converts filter object from the queryObject to the API's must-include format. + * Format: field1,value1,value2;field2,value1,value2 + * @param {Object} queryObject - The query object containing filter criteria. + * @returns {string} - Formatted filter string for the API. + */ +function buildMustIncludeParams(queryObject) { + const filters = []; + + // Handle tags filter + if (queryObject.tags && queryObject.tags.length > 0) { + filters.push(`tags,${queryObject.tags.join(',')}`); + } + + // Handle gem5_versions filter + if (queryObject.gem5_versions && queryObject.gem5_versions.length > 0) { + filters.push(`gem5_versions,${queryObject.gem5_versions.join(',')}`); + } + + // Handle category filter + if (queryObject.category && queryObject.category.length > 0) { + filters.push(`category,${queryObject.category.join(',')}`); + } + + // Handle architecture filter + if (queryObject.architecture && queryObject.architecture.length > 0) { + filters.push(`architecture,${queryObject.architecture.join(',')}`); + } + + return filters.join(';'); } /** @@ -366,14 +82,8 @@ function getPipeline(queryObject, currentPage, pageSize) { * @description This asynchronous function fetches search results from a specified data source using MongoDB aggregation pipeline. * It takes in an access token, URL, data source, database, collection, query object, current page number, and page size as input parameters, * and returns an array containing the search results and total count of documents matching the search criteria. - * @param {string} accessToken - The access token for authentication. * @param {string} url - The URL of the data source. - * @param {string} dataSource - The name of the data source. - * @param {string} database - The name of the database. - * @param {string} collection - The name of the collection. * @param {Object} queryObject - The query object containing search query, sort, and filter parameters. - * @param {string} queryObject.query - The search query. - * @param {string} queryObject.sort - The sorting parameter. * @param {number} currentPage - The current page number. * @param {number} pageSize - The number of documents to display per page. * @returns {Array} - An array containing the search results and total count of documents matching the search criteria. @@ -381,42 +91,48 @@ function getPipeline(queryObject, currentPage, pageSize) { async function getSearchResults( accessToken, url, - dataSource, - database, - collection, queryObject, currentPage, pageSize ) { - let resources = []; - const pipeline = getPipeline(queryObject, currentPage, pageSize); - const res = await fetch(`${url}/action/aggregate`, { - method: "POST", - headers: { - "Content-Type": "application/json", - "Access-Control-Request-Headers": "*", - Authorization: "Bearer " + accessToken, - }, - // also apply filters on - body: JSON.stringify({ - dataSource: dataSource, - database: database, - collection: collection, - pipeline: pipeline, - }), - }).catch((err) => console.log(err)); - resources = await res.json(); - // check if status is not 200 - if (res.status !== 200) { - console.log("Error: " + res.status); + try { + // Build the query parameters + const params = getPipeline(queryObject, currentPage, pageSize); + // Make the API call to the new endpoint + const res = await fetch(`${url}/search?${params.toString()}`, { + method: "GET", + headers: { + "Content-Type": "application/json", + // "x-functions-key": accessToken, // Using API token as x-functions-key for Azure Functions + }, + }); + + // Check if status is not 200 + if (res.status !== 200) { + console.log("Error: " + res.status); + return [[], 0]; + } + + // Parse the response + const responseData = await res.json(); + let documents = responseData.documents + console.log(documents) + // If there are no results, return empty array and count 0 + if (!documents || !Array.isArray(documents) || documents.length === 0) { + return [[], 0]; + } + + // Calculate the total count + // Note: The API might return the total count in a different format + // If the API returns total count separately, this logic should be updated + const totalCount = responseData.totalCount; + + // Return in the same format as the original function + return [documents, totalCount]; + } catch (err) { + console.log("Error fetching resources:", err); return [[], 0]; } - return [ - resources["documents"], - resources["documents"].length > 0 - ? resources["documents"][0].totalCount - : 0, - ]; } @@ -434,9 +150,6 @@ export default async function getResourcesByQueryMongoDB(queryObject, currentPag let privateAccessToken = await getToken(database); let privateResourceResults = await getSearchResults(privateAccessToken, privateResource.url, - privateResource.dataSource, - privateResource.database, - privateResource.collection, queryObject, currentPage, pageSize); diff --git a/pages/api/mongodb/getVersionsByID.js b/pages/api/mongodb/getVersionsByID.js index e6f1bdf..444ff27 100644 --- a/pages/api/mongodb/getVersionsByID.js +++ b/pages/api/mongodb/getVersionsByID.js @@ -14,30 +14,22 @@ import compareVersions from "../compareVersions"; * @returns {Array} - An array of resource versions retrieved from the specified data source, database, and collection, sorted in descending order of resource version. * @throws {Error} - Throws an error if the fetch request fails. */ -async function getVersionsByID(token, url, dataSource, database, collection, id) { - const res = await fetch(`${url}/action/find`, { - method: 'POST', +async function getVersionsByID(token, url, id) { + const res = await fetch(`${url}/find-resource-by-id?id=${id}`, { + method: 'GET', headers: { 'Content-Type': 'application/json', // 'api-key': 'pKkhRJGJaQ3NdJyDt69u4GPGQTDUIhHlx4a3lrKUNx2hxuc8uba8NrP3IVRvlzlo', - 'Access-Control-Request-Headers': '*', + // 'Access-Control-Request-Headers': '*', // 'origin': 'https://gem5vision.github.io', - "Authorization": "Bearer " + token, + }, - body: JSON.stringify({ - "dataSource": dataSource, - "database": database, - "collection": collection, - "filter": { - "id": id - } - }) }).catch(err => console.log(err)); let resource = await res.json(); - if (res.status != 200 || resource['documents'] === null) { + if (res.status != 200 || resource === null) { return { error: 'Resource not found' } } - resource = resource['documents'].sort((a, b) => -compareVersions(a.resource_version, b.resource_version)); + resource = resource.sort((a, b) => -compareVersions(a.resource_version, b.resource_version)); return resource; } @@ -55,7 +47,7 @@ export default async function getVersionsByIDMongoDB(id, database = null) { } const token = await getToken(database); let privateResources = process.env.SOURCES[database]; - const resource = await getVersionsByID(token, privateResources.url, privateResources.dataSource, privateResources.database, privateResources.collection, id); + const resource = await getVersionsByID(token, privateResources.url, id); resource.forEach(res => { res['database'] = database; });