Skip to content
Snippets Groups Projects
Unverified Commit 1746e49c authored by Rodrigo Nascimento's avatar Rodrigo Nascimento Committed by GitHub
Browse files

refactor: Remove promise.await from csv importer (#28810)

parent 958904b4
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
apps/meteor/app/importer-csv/server/importer.js
+ 143
− 147
View file @ 1746e49c
...@@ -53,173 +53,169 @@ export class CsvImporter extends Base { ...@@ -53,173 +53,169 @@ export class CsvImporter extends Base {
return roomIds.get(roomName); return roomIds.get(roomName);
}; };
zip.forEach((entry) => { for await (const entry of zip.getEntries()) {
Promise.await( this.logger.debug(`Entry: ${entry.entryName}`);
(async () => {
this.logger.debug(`Entry: ${entry.entryName}`);
// Ignore anything that has `__MACOSX` in it's name, as sadly these things seem to mess everything up
if (entry.entryName.indexOf('__MACOSX') > -1) {
this.logger.debug(`Ignoring the file: ${entry.entryName}`);
return increaseProgressCount();
}
// Directories are ignored, since they are "virtual" in a zip file // Ignore anything that has `__MACOSX` in it's name, as sadly these things seem to mess everything up
if (entry.isDirectory) { if (entry.entryName.indexOf('__MACOSX') > -1) {
this.logger.debug(`Ignoring the directory entry: ${entry.entryName}`); this.logger.debug(`Ignoring the file: ${entry.entryName}`);
return increaseProgressCount(); return increaseProgressCount();
} }
// Parse the channels // Directories are ignored, since they are "virtual" in a zip file
if (entry.entryName.toLowerCase() === 'channels.csv') { if (entry.isDirectory) {
await super.updateProgress(ProgressStep.PREPARING_CHANNELS); this.logger.debug(`Ignoring the directory entry: ${entry.entryName}`);
const parsedChannels = this.csvParser(entry.getData().toString()); return increaseProgressCount();
channelsCount = parsedChannels.length; }
for await (const c of parsedChannels) {
const name = c[0].trim();
const id = getRoomId(name);
const creator = c[1].trim();
const isPrivate = c[2].trim().toLowerCase() === 'private';
const members = c[3]
.trim()
.split(';')
.map((m) => m.trim())
.filter((m) => m);
await this.converter.addChannel({ // Parse the channels
importIds: [id], if (entry.entryName.toLowerCase() === 'channels.csv') {
u: { await super.updateProgress(ProgressStep.PREPARING_CHANNELS);
_id: creator, const parsedChannels = this.csvParser(entry.getData().toString());
}, channelsCount = parsedChannels.length;
name,
users: members, for await (const c of parsedChannels) {
t: isPrivate ? 'p' : 'c', const name = c[0].trim();
}); const id = getRoomId(name);
} const creator = c[1].trim();
const isPrivate = c[2].trim().toLowerCase() === 'private';
const members = c[3]
.trim()
.split(';')
.map((m) => m.trim())
.filter((m) => m);
await this.converter.addChannel({
importIds: [id],
u: {
_id: creator,
},
name,
users: members,
t: isPrivate ? 'p' : 'c',
});
}
await super.updateRecord({ 'count.channels': channelsCount }); await super.updateRecord({ 'count.channels': channelsCount });
return increaseProgressCount(); return increaseProgressCount();
} }
// Parse the users // Parse the users
if (entry.entryName.toLowerCase() === 'users.csv') { if (entry.entryName.toLowerCase() === 'users.csv') {
await super.updateProgress(ProgressStep.PREPARING_USERS); await super.updateProgress(ProgressStep.PREPARING_USERS);
const parsedUsers = this.csvParser(entry.getData().toString()); const parsedUsers = this.csvParser(entry.getData().toString());
usersCount = parsedUsers.length; usersCount = parsedUsers.length;
for await (const u of parsedUsers) {
const username = u[0].trim();
availableUsernames.add(username);
const email = u[1].trim();
const name = u[2].trim();
await this.converter.addUser({
importIds: [username],
emails: [email],
username,
name,
});
}
for await (const u of parsedUsers) { await super.updateRecord({ 'count.users': usersCount });
const username = u[0].trim(); return increaseProgressCount();
availableUsernames.add(username); }
const email = u[1].trim(); // Parse the messages
const name = u[2].trim(); if (entry.entryName.indexOf('/') > -1) {
if (this.progress.step !== ProgressStep.PREPARING_MESSAGES) {
await super.updateProgress(ProgressStep.PREPARING_MESSAGES);
}
await this.converter.addUser({ const item = entry.entryName.split('/'); // random/messages.csv
importIds: [username], const folderName = item[0]; // random
emails: [email],
username,
name,
});
}
await super.updateRecord({ 'count.users': usersCount }); let msgs = [];
return increaseProgressCount();
}
// Parse the messages try {
if (entry.entryName.indexOf('/') > -1) { msgs = this.csvParser(entry.getData().toString());
if (this.progress.step !== ProgressStep.PREPARING_MESSAGES) { } catch (e) {
await super.updateProgress(ProgressStep.PREPARING_MESSAGES); this.logger.warn(`The file ${entry.entryName} contains invalid syntax`, e);
} return increaseProgressCount();
}
const item = entry.entryName.split('/'); // random/messages.csv let data;
const folderName = item[0]; // random const msgGroupData = item[1].split('.')[0]; // messages
let isDirect = false;
if (folderName.toLowerCase() === 'directmessages') {
isDirect = true;
data = msgs.map((m) => ({
username: m[0],
ts: m[2],
text: m[3],
otherUsername: m[1],
isDirect: true,
}));
} else {
data = msgs.map((m) => ({ username: m[0], ts: m[1], text: m[2] }));
}
let msgs = []; messagesCount += data.length;
const channelName = `${folderName}/${msgGroupData}`;
try { await super.updateRecord({ messagesstatus: channelName });
msgs = this.csvParser(entry.getData().toString());
} catch (e) {
this.logger.warn(`The file ${entry.entryName} contains invalid syntax`, e);
return increaseProgressCount();
}
let data; if (isDirect) {
const msgGroupData = item[1].split('.')[0]; // messages for await (const msg of data) {
let isDirect = false; const sourceId = [msg.username, msg.otherUsername].sort().join('/');
if (folderName.toLowerCase() === 'directmessages') { if (!dmRooms.has(sourceId)) {
isDirect = true; await this.converter.addChannel({
data = msgs.map((m) => ({ importIds: [sourceId],
username: m[0], users: [msg.username, msg.otherUsername],
ts: m[2], t: 'd',
text: m[3], });
otherUsername: m[1],
isDirect: true,
}));
} else {
data = msgs.map((m) => ({ username: m[0], ts: m[1], text: m[2] }));
}
messagesCount += data.length; dmRooms.set(sourceId, true);
const channelName = `${folderName}/${msgGroupData}`;
await super.updateRecord({ messagesstatus: channelName });
if (isDirect) {
for await (const msg of data) {
const sourceId = [msg.username, msg.otherUsername].sort().join('/');
if (!dmRooms.has(sourceId)) {
await this.converter.addChannel({
importIds: [sourceId],
users: [msg.username, msg.otherUsername],
t: 'd',
});
dmRooms.set(sourceId, true);
}
const newMessage = {
rid: sourceId,
u: {
_id: msg.username,
},
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
};
usedUsernames.add(msg.username);
usedUsernames.add(msg.otherUsername);
await this.converter.addMessage(newMessage);
}
} else {
const rid = getRoomId(folderName);
for await (const msg of data) {
const newMessage = {
rid,
u: {
_id: msg.username,
},
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
};
usedUsernames.add(msg.username);
await this.converter.addMessage(newMessage);
}
} }
await super.updateRecord({ 'count.messages': messagesCount, 'messagesstatus': null }); const newMessage = {
return increaseProgressCount(); rid: sourceId,
u: {
_id: msg.username,
},
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
};
usedUsernames.add(msg.username);
usedUsernames.add(msg.otherUsername);
await this.converter.addMessage(newMessage);
} }
})(), } else {
); const rid = getRoomId(folderName);
for await (const msg of data) {
const newMessage = {
rid,
u: {
_id: msg.username,
},
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
};
usedUsernames.add(msg.username);
await this.converter.addMessage(newMessage);
}
}
await super.updateRecord({ 'count.messages': messagesCount, 'messagesstatus': null });
return increaseProgressCount();
}
increaseProgressCount(); increaseProgressCount();
}); }
if (usersCount) { if (usersCount) {
await Settings.incrementValueById('CSV_Importer_Count', usersCount); await Settings.incrementValueById('CSV_Importer_Count', usersCount);
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment