[SOLVED] Pass response from puppeter to API

Issue

I want to open the link from puppeteer and get API link from network and pass to our API after every 3 minutes.

Error If I console the link variable it will show the link but in return it's empty

leader

const puppeteer = require('puppeteer');

let link = '';
var url = async () => {

const browser = await puppeteer.launch({
    headless: false
})
const page = await browser.newPage()
await page.goto("https://www.pgatour.com/leaderboard.html", {
    waitUntil: 'networkidle2'
});

let urls = await page.on('response', async (response) => {
    if (response.url().includes("leaderboard.json")) {
        console.log('XHR response received');
        const data = await response.buffer()
        link = response.url(); // ========> link get if I console it
        browser.close();

    }
});

return link;  //==================== PROBLEM================= link is empty
}

cron.js cron which will open the link after 3 minute

const cron = require("node-cron");
const axios = require("axios");
var leader = require('leader');

cron.schedule("* 3 * * *", function () {
console.log("running a task 3 minute")
axios.get('/link', {
    params: {
      link: leader  //<<<< ========================== here I want to pass the link from leader.js file
    }
  })
  .then(function (response) {
    console.log(response);
  })
  .catch(function (error) {
    console.log(error);
  }) 
})

I am a beginner in node js so I don't know how to get data from puppeteer and pass to our function

Solution

If you create a getLeader() function in a leader.js module like so:

leader.js

const puppeteer = require('puppeteer');

async function getLeader() {
    const browser = await puppeteer.launch({
        headless: true
    })
    console.log("Loading page...");
    const page = await browser.newPage()
    await page.goto("https://www.pgatour.com/leaderboard.html", {
        waitUntil: 'networkidle2'
    });

    return new Promise((resolve, reject) => { 
        page.on('response', async (response) => {
            if (response.url().includes("leaderboard.json")) {
                console.log('XHR response received');
                link = response.url(); // ========> link get if I console it
                browser.close();
                console.log("Link:", link);
                resolve(link);
            }
        });
    })
}

module.exports = { getLeader };

Then in your cron module:

cron

const cron = require("node-cron");
const axios = require("axios");
const { getLeader } = require('./leader.js');

cron.schedule("*/3 * * * *", async function () {
    console.log("running a task every 3 minutes");

    let leader = await getLeader();
    console.log("Cron job: Leader:", leader);
    axios.get('/link', {
        params: {
            link: leader  //<<<< ========================== here I want to pass the link from leader.js file
        }
    })
    .then(function (response) {
        console.log(response);
    })
    .catch(function (error) {
        console.log(error);
    }) 
})

If you start the cron script this will pull the leader every 3 minutes.

Answered By - Terry Lennox

Answer Checked By - Terry (WPSolving Volunteer)