瀏覽代碼

Faster: serialise fetch, save on background.

Herby Vojčík 4 年之前
父節點
當前提交
3c85963032
共有 1 個文件被更改,包括 4 次插入2 次删除
  1. 4 2
      index.js

+ 4 - 2
index.js

@@ -9,16 +9,18 @@ async function work () {
     console.info(`Datasets: ${count}`);
     for (let {title, num_resources, resources} of results) {
         console.info(`Fetching: ${title}, ${num_resources} files`);
+        const saves = [];
         for (let {name, description, url} of resources) {
             const [, issue, year] = name.match(/(\d+)\/(\d+)/);
             console.info(`File ${year} / ${issue}: ${description}`);
             const xmlRaw = await fetch(url);
-            await new Promise((resolve, reject) =>
+            saves.push(new Promise((resolve, reject) =>
                 xmlRaw.body.pipe(createWriteStream(`${year}.${issue}.xml`))
                     .on('close', resolve)
                     .on('error', reject)
-            );
+            ));
         }
+        await Promise.all(saves);
     }
     return results;
 }