fs.writeFile 在写入第一个 json 文件后导致节点应用程序崩溃

vo-*_*-so 8 fs node.js promise bluebird puppeteer

我试图爬行几个网页来检查损坏的链接并将链接的结果写入 json 文件,但是,在第一个文件完成后,应用程序崩溃,没有弹出错误...

我使用 Puppeteer 进行爬行,使用 Bluebird 来同时运行每个链接,使用 fs 来写入文件。

我尝试过什么:

  • 将文件类型切换为“.txt”或“.php”,这可行,但我需要在当前工作流程之外创建另一个循环,以将文件从“.txt”转换为“.json”。在写入文件后立即重命名该文件也会导致应用程序崩溃。
  • 对 fs.writeFile 使用 try catch 语句但它永远不会引发错误
  • Express之外的整个应用程序,这在某些时候有效,但我尝试在框架内使用它
const express = require('express');
const router = express.Router();
const puppeteer = require('puppeteer');
const bluebird = require("bluebird");
const fs = require('fs');

router.get('/', function(req, res, next) {
    (async () => {
        // Our (multiple) URLs.
        const urls = ['https://www.testing.com/allergy-test/', 'https://www.testing.com/genetic-testing/'];
    
        const withBrowser = async (fn) => {
            const browser = await puppeteer.launch();
            try {
                return await fn(browser);
            } finally {
                await browser.close();
            }
        }
    
        const withPage = (browser) => async (fn) => {
            const page = await browser.newPage();

            // Turns request interceptor on.
            await page.setRequestInterception(true);

            // Ignore all the asset requests, just get the document.
            page.on('request', request => {
                if (request.resourceType() === 'document' ) {
                    request.continue();
                } else {
                    request.abort();
                }
            });

            try {
                return await fn(page);
            } finally {
                await page.close();
            }
        }
    
        const results = await withBrowser(async (browser) => {
            return bluebird.map(urls, async (url) => {
                return withPage(browser)(async (page) => {                    
                    await page.goto(url, {
                        waitUntil: 'domcontentloaded',
                        timeout: 0 // Removes timeout.
                    });
    
                    // Search for urls we want to "crawl".
                    const hrefs = await page.$$eval('a[href^="https://www.testing.com/"]', as => as.map(a => a.href));

                    // Predefine our arrays.
                    let links = [];
                    let redirect = [];
    
                    // Loops through each /goto/ url on page
                    for (const href of Object.entries(hrefs)) {
                        response = await page.goto(href[1], {
                            waitUntil: 'domcontentloaded',
                            timeout: 0 // Remove timeout.
                        });
                        const chain = response.request().redirectChain();
    
                        const link = {      
                            'source_url': href[1],
                            'status': response.status(),
                            'final_url': response.url(),
                            'redirect_count': chain.length,
                        };
                        // Loops through the redirect chain for each href.
                        for ( const ch of chain) {
                            redirect = {
                                status: ch.response().status(),
                                url: ch.url(),
                            };
                        }

                        // Push all info of target link into links
                        links.push(link);
                    }  
                    // JSONify the data.
                    const linksJson = JSON.stringify(links);

                    fileName = url.replace('https://www.testing.com/', '');
                    fileName = fileName.replace(/[^a-zA-Z0-9\-]/g, '');
                        
                    // Write data to file in /tmp directory.
                    fs.writeFile(`./tmp/${fileName}.json`, linksJson, (err) => {
                        if (err) {
                            return console.log(err);
                        }
                    });
                });
            }, {concurrency: 4}); // How many pages to run at a time.
        });
    })();   
});
module.exports = router;
Run Code Online (Sandbox Code Playgroud)

更新:所以我的代码没有任何问题...我意识到nodemon在保存每个文件后停止了进程。由于nodemon会检测到“文件更改”,因此它在第一个项目之后不断重新启动我的服务器