filecoin-project/slate
1
1
Fork 0
mirror of https://github.com/filecoin-project/slate.git synced 2024-11-22 12:24:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

fixing some places where search manager wasn't properly updating files

Browse Source
This commit is contained in:
Martina 2021-11-03 10:56:42 -07:00
parent 2d33c0ef47
commit 3247cff0d1
12 changed files with 122 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
node_common/managers/search/index.js
View File

@ -11,6 +11,9 @@ import {
searchUser,
searchSlate,
searchFile,
getUser,
getSlate,
getFile,
} from "~/node_common/managers/search/search";
import {
indexUser,
@ -44,6 +47,9 @@ const SearchManager = {
searchUser,
searchSlate,
searchFile,
getUser,
getSlate,
getFile,
};
export default SearchManager;

72
node_common/managers/search/manage.js
View File

@ -98,6 +98,47 @@ export const createSlateIndex = async () => {
};
export const createFileIndex = async () => {
let props = {
settings: {
number_of_shards: 1,
number_of_replicas: 0,
index: {
analysis: {
char_filter: {
my_pattern: {
type: "pattern_replace",
pattern: "a",
replacement: "u",
},
},
analyser: {
my_analyser: {
type: "custom",
tokenizer: "whitespace",
char_filter: ["my_pattern"],
},
},
},
},
},
mappings: {
my_type: {
_source: {
enabled: true,
},
},
},
properties: {
test: {
type: "string",
store: true,
index: "analysed",
analyser: "my_analyser",
index_options: "positions",
},
},
};
let properties = {
mappings: {
properties: {
@ -251,34 +292,3 @@ export const deleteFileIndex = async () => {
console.log(e);
}
};
//make it synonym search
//make searchall work across the different types
//pagination
//use index = _all or empty string to perform the operation on all indices
//specify from and size to paginate (from = 0, size = 10 by default)
//make sure that for the non index fields, you aren't able to search by them
//make sure that for slatename type hyphenated things, you're able to search for them as titles with sapces. treat hyphens like spaces
//should we do filetype filtering and isLink / isNotLink on the front end? If so we don't have to index them here on the backend
//if you index again and leave something out, it'll then remove that. it basically overwrites the existing one
//what about update? does that ignore things that are left out? what if you try to update something that doesn't exist?
// { "range": { "publish_date": { "gte": "2015-01-01", "lte": ... }}}
//make it so it can find foggy.jpeg with just foggy. Either treat periods as spaces, or be able to search incomplete queries (wild card search). maybe using tokenizer?
//how to make this work with checking a value INSIDE the tags object? do we need to change the shape of the tags object?
//add auto suggest
//did boost work?
//sort search results by whether it's your result or another user's
//test out slates and files add and search
//make sure searching with spaces in a hyphenated slate name works
//make sure filtering by tag works for files
//test out delete
//detail all the filtering types and make convenience functions for them
//sort by ownerId match first?

19
node_common/managers/search/search.js
View File

@ -233,7 +233,6 @@ export const searchFile = async ({ query, userId, globalSearch = false, tagIds =
const must = [
{
multi_match: {
//or maybe do combined_fields
query,
fuzziness: "AUTO",
type: "best_fields",
@ -343,3 +342,21 @@ export const searchFile = async ({ query, userId, globalSearch = false, tagIds =
console.log(e);
}
};
export const getUser = async ({ id }) => {
const { body } = await searchClient.get({ index: usersIndex, id });
console.log(body);
return body;
};
export const getSlate = async ({ id }) => {
const { body } = await searchClient.get({ index: slatesIndex, id });
console.log(body);
return body;
};
export const getFile = async ({ id }) => {
const { body } = await searchClient.get({ index: filesIndex, id });
console.log(body);
return body;
};

9
node_common/managers/search/update.js
View File

@ -106,6 +106,9 @@ const cleanFile = ({
};
const indexObject = async (objects, cleanObject, index) => {
console.log("index object");
console.log(objects);
console.log(index);
try {
if (Array.isArray(objects)) {
let body = [];
@ -145,6 +148,9 @@ export const indexFile = async (files) => {
};
const updateObject = async (objects, cleanObject, index) => {
console.log("update object");
console.log(objects);
console.log(index);
try {
if (Array.isArray(objects)) {
let body = [];
@ -185,6 +191,9 @@ export const updateFile = async (files) => {
};
const deleteObject = async (objects, index) => {
console.log("delete object");
console.log(objects);
console.log(index);
try {
if (Array.isArray(objects)) {
let body = [];

6
node_common/utilities.js
View File

@ -386,9 +386,6 @@ export const removeFromPublicCollectionUpdatePrivacy = async ({ files }) => {
madePrivate.push(updatedFile);
}
}
if (madePrivate.length) {
SearchManager.updateFile(madePrivate);
}
return madePrivate;
};
@ -402,8 +399,5 @@ export const addToPublicCollectionUpdatePrivacy = async ({ files }) => {
madePublic.push(updatedFile);
}
}
if (madePublic.length) {
SearchManager.updateFile(madePublic);
}
return madePublic;
};

7
pages/api/data/save-copy.js
View File

@ -105,13 +105,14 @@ export default async (req, res) => {
added = addedToSlate;
}
SearchManager.createFile(createdFiles);
//NOTE(martina): leaving createdFiles out of the privacy recalculation since those should already have the correct privacy
if (slate?.isPublic) {
Utilities.addToPublicCollectionUpdatePrivacy({ files: duplicateFiles });
await Utilities.addToPublicCollectionUpdatePrivacy({ files: duplicateFiles });
}
let updatedFiles = await Data.getFilesByIds({ ids: filesToAddToSlate.map((file) => file.id) });
SearchManager.indexFile(updatedFiles); //NOTE(martina): using createFile instead of updateFile b/c createFile also works for existing files (it just overwrites)
ViewerManager.hydratePartial(id, { library: true, slates: slate ? true : false });
if (!slate) {

5
pages/api/slates/delete.js
View File

@ -39,7 +39,10 @@ export default async (req, res) => {
SearchManager.deleteSlate(slate);
if (slate.isPublic) {
Utilities.removeFromPublicCollectionUpdatePrivacy({ files: slate.objects });
let updatedFiles = await Utilities.removeFromPublicCollectionUpdatePrivacy({
files: slate.objects,
});
SearchManager.updateFile(updatedFiles);
}
return res.status(200).send({ decorator: "SERVER_DELETE_SLATE", error: false });

5
pages/api/slates/remove-file.js
View File

@ -52,7 +52,10 @@ export default async (req, res) => {
}
if (slate.isPublic) {
Utilities.removeFromPublicCollectionUpdatePrivacy({ files: slate.objects });
let updatedFiles = await Utilities.removeFromPublicCollectionUpdatePrivacy({
files: slate.objects,
});
SearchManager.updateFile(updatedFiles);
}
ViewerManager.hydratePartial(id, { slates: true });

9
pages/api/slates/update.js
View File

@ -97,11 +97,16 @@ export default async (req, res) => {
SearchManager.updateSlate(response);
let updatedFiles;
if (slate.isPublic && !updates.isPublic) {
Utilities.removeFromPublicCollectionUpdatePrivacy({ files: slate.objects });
updatedFiles = await Utilities.removeFromPublicCollectionUpdatePrivacy({
files: slate.objects,
});
} else if (!slate.isPublic && updates.isPublic) {
Utilities.addToPublicCollectionUpdatePrivacy({ files: slate.objects });
updatedFiles = await Utilities.addToPublicCollectionUpdatePrivacy({ files: slate.objects });
}
SearchManager.updateFile(updatedFiles);
return res.status(200).send({ decorator: "SERVER_UPDATE_SLATE", slate: response });
};

9
pages/api/v1/update-slate.js
View File

@ -61,11 +61,16 @@ export default async (req, res) => {
return res.status(500).send({ decorator: "UPDATE_SLATE_PRIVACY_FAILED", error: true });
}
let updatedFiles;
if (!updates.isPublic) {
Utilities.removeFromPublicCollectionUpdatePrivacy({ files: slate.objects });
updatedFiles = await Utilities.removeFromPublicCollectionUpdatePrivacy({
files: slate.objects,
});
} else {
Utilities.addToPublicCollectionUpdatePrivacy({ files: slate.objects });
updatedFiles = await Utilities.addToPublicCollectionUpdatePrivacy({ files: slate.objects });
}
SearchManager.updateFile(updatedFiles);
}
if (updates.data.name && updates.data.name !== slate.data.name) {

8
pages/api/v2/update-collection.js
View File

@ -94,11 +94,15 @@ export default async (req, res) => {
return res.status(500).send({ decorator: "UPDATE_COLLECTION_FAILED", error: true });
}
let updatedFiles;
if (slate.isPublic && !updates.isPublic) {
Utilities.removeFromPublicCollectionUpdatePrivacy({ files: slate.objects });
updatedFiles = await Utilities.removeFromPublicCollectionUpdatePrivacy({
files: slate.objects,
});
} else if (!slate.isPublic && updates.isPublic) {
Utilities.addToPublicCollectionUpdatePrivacy({ files: slate.objects });
updatedFiles = await Utilities.addToPublicCollectionUpdatePrivacy({ files: slate.objects });
}
SearchManager.updateFile(updatedFiles);
SearchManager.updateSlate(updatedSlate);

29
scripts/search.js
View File

@ -139,18 +139,21 @@ async function update() {
async function search() {
// await SearchManager.searchUser({ query: "image" });
// await SearchManager.searchSlate({
// query: "slate",
// userId: "5172dd8b-6b11-40d3-8c9f-b4cbaa0eb8e7",
// globalSearch: true,
// });
// await SearchManager.searchFile({
// query: "file",
// userId: "f9cc7b00-ce59-4b49-abd1-c7ef7253e258",
// globalSearch: true,
// tagIds: ["d82fbc78-88de-4015-adec-a7ea832fc922", "0824a3cb-e839-4246-8ff4-d919919e1487"],
// });
// await SearchManager.searchSlate({
// query: "my slate",
// userId: "5172dd8b-6b11-40d3-8c9f-b4cbaa0eb8e7",
// globalSearch: true,
// });
await SearchManager.searchFile({
query: "grilled cheese",
userId: "f9cc7b00-ce59-4b49-abd1-c7ef7253e258",
globalSearch: true,
tagIds: [],
// tagIds: ["d82fbc78-88de-4015-adec-a7ea832fc922", "0824a3cb-e839-4246-8ff4-d919919e1487"],
});
// await SearchManager.searchAll({ query: "slate", userId: "5172dd8b-6b11-40d3-8c9f-b4cbaa0eb8e7" });
// await SearchManager.getFile({ id: "2892b652-5034-4e0f-b3b2-0352e0d64e17" });
}
async function setUpIndex() {
@ -169,8 +172,8 @@ async function resetIndex() {
await setUpIndex();
}
setUpIndex();
// Promise.all([manage(), update(), search()]);
// setUpIndex();
Promise.all([manage(), update(), search()]);
Logging.log(`FINISHED: search.js`);
Logging.log(` CTRL + C to return to terminal.`);