buildInfoCrawler/index.js

import axios, { all } from "axios"
import pRetry, { AbortError } from "p-retry"
import { parse } from "node-html-parser"
import tmp from "tmp"
import fs, { readFileSync } from "fs"
import path from "path"
import { execFileSync, spawn } from "child_process"
import printf from "printf"
import chalk from "chalk"
import PQueue from "p-queue"
import { setTimeout } from "timers/promises"

async function fetchModels() {
    console.log("fetching models")
    const html = await pRetry(
        async () => {
            const response = await fetch("https://desktop.firmware.mobi")

            // Abort retrying if the resource doesn't exist
            if (response.status === 404) {
                throw new AbortError(response.statusText)
            }

            return response.text()
        },
        { retries: 5 }
    )
    const root = parse(html)
    const scripts = root.querySelectorAll("script")
    let allModels = []
    scripts.map(s => {
        const text = s.text.trim()
        if (!text.includes("models =")) {
            return
        }
        let models = []
        let firmwares
        eval(s.text)
        allModels = allModels.concat(models)
    })
    allModels.sort((a, b) => a.id - b.id)
    console.log(`total models: ${allModels.length}`)

    return allModels
}

async function fetchFirmwares(model) {
    console.log(`fetching firmwares: ${model.de}(modelId=${model.id})`)
    const html = await pRetry(
        async () => {
            const response = await fetch(
                `https://desktop.firmware.mobi/device:${model.id}`
            )

            // Abort retrying if the resource doesn't exist
            if (response.status === 404) {
                throw new AbortError(response.statusText)
            }

            return response.text()
        },
        { retries: 5 }
    )
    const root = parse(html)
    const scripts = root.querySelectorAll("script")
    let allFirmwares = []
    scripts.map(s => {
        const text = s.text.trim()
        if (!text.includes("firmwares =")) {
            return
        }
        let firmwares = []
        eval(s.text)
        allFirmwares = allFirmwares.concat(firmwares)
    })
    console.log(`${allFirmwares.length} firmwares found`)
    model.firmwares = allFirmwares
    return allFirmwares
}

async function fetchProps(model, firmware) {
    console.log(
        `fetching props: ${model.de}(modelId=${model.id}, firmwareId=${firmware.id})`
    )
    const html = await pRetry(
        async () => {
            const response = await fetch(
                `https://desktop.firmware.mobi/device:${model.id}/firmware:${firmware.id}`
            )

            // Abort retrying if the resource doesn't exist
            if (response.status === 404) {
                throw new AbortError(response.statusText)
            }

            return response.text()
        },
        { retries: 5 }
    )
    const root = parse(html)
    const rows = root.querySelectorAll(".rTableRow")
    const props = {}
    rows.map(row => {
        const cells = row.querySelectorAll(".rTableCell")
        if (cells.length === 2) {
            return {
                key: cells[0].text.replace(/^- /, "").trim(),
                value: cells[1].text.trim()
            }
        }
        return null
    })
        .filter(i => {
            return i && i.key && i.value
        })
        .forEach(i => {
            props[i.key] = i.value
        })
    console.log(props)

    const collaps = root.querySelectorAll(".collapsible-body")
    const rawProps = collaps
        .map(c =>
            c
                .querySelectorAll("pre")
                .map(pre => pre.text)
                .join("")
        )
        .join()

    const tmpDir = tmp.dirSync()
    console.log(tmpDir.name)

    fs.copyFileSync(
        path.join("scripts", "gen_pif_custom.sh"),
        path.join(tmpDir.name, "gen_pif_custom.sh")
    )
    fs.chmodSync(path.join(tmpDir.name, "gen_pif_custom.sh"), "755")

    let buildProp = ""
    let vendorBuildProp = ""
    let productBuildProp = ""
    rawProps.split("\n").forEach(line => {
        if (line.includes(".vendor.")) {
            vendorBuildProp += line + "\n"
        } else if (
            line.includes(".product.product.") ||
            line.includes(".product.build.") ||
            line.includes(".product.vndk.")
        ) {
            productBuildProp += line + "\n"
        } else {
            buildProp += line + "\n"
        }
    })
    if (!buildProp.includes("ro.product.system.device=generic")) {
        fs.writeFileSync(path.join(tmpDir.name, "build.prop"), buildProp)
    }
    fs.writeFileSync(
        path.join(tmpDir.name, "vendor-build.prop"),
        vendorBuildProp
    )
    fs.writeFileSync(
        path.join(tmpDir.name, "product-build.prop"),
        productBuildProp
    )

    try {
        // execFileSync(path.join(tmpDir.name, "gen_pif_custom.sh"), [], {
        //     cwd: tmpDir.name
        // })

        await new Promise((resolve, reject) => {
            const p = spawn(path.join(tmpDir.name, "gen_pif_custom.sh"), {
                cwd: tmpDir.name
            })
            p.stdout.on("data", data => {
                console.log(chalk.cyan(data.toString().replace("\n", "")))
            })
            p.stderr.on("data", data => {
                console.log(chalk.red(data.toString().replace("\n", "")))
            })
            p.on("close", code => {
                resolve()
            })
        })
    } catch (e) {}

    const pifPath = path.join(tmpDir.name, "custom.pif.json")
    if (fs.existsSync(pifPath)) {
        fs.copyFileSync(
            pifPath,
            path.join(
                "dist",
                `${printf("%05d", model.id)}_${printf(
                    "%05d",
                    firmware.id
                )}.json`
            )
        )
    }
}
if (!fs.existsSync("dist")) {
    fs.mkdirSync("dist")
}

const queue = new PQueue({ concurrency: 2 })

// const models = await fetchModels()
// models.forEach(model => {
//     queue.add(() => fetchFirmwares(model))
// })
// await queue.onIdle()
// fs.writeFileSync("models.json", JSON.stringify(models, null, 2))
const models = JSON.parse(fs.readFileSync("models.json").toString())

// models.forEach(model => {
//     model.firmwares.forEach(firmware => {
//         queue.add(() => fetchProps(model, firmware))
//     })
// })
// await queue.onIdle()

console.log(
    models.map(model => model.firmwares.length).reduce((a, b) => a + b, 0)
)

const missingModels = []
const missingFirmwares = []
models.forEach(model => {
    model.firmwares.forEach(firmware => {
        const pifPath = path.join(
            "dist",
            `${printf("%05d", model.id)}_${printf("%05d", firmware.id)}.json`
        )
        if (!fs.existsSync(pifPath)) {
            if (missingModels.findIndex(m => m.id === model.id) === -1) {
                missingModels.push(model)
                missingFirmwares.push([model, firmware])
            }
        }
    })
})
console.log(`missing: ${missingModels.length}`)
missingFirmwares.sort((a, b) => a[0].id - b[0].id)

missingFirmwares.forEach(([model, firmware]) => {
    console.log(
        `https://desktop.firmware.mobi/device:${model.id}/firmware:${firmware.id}`
    )
})

missingFirmwares
    .reverse()
    .forEach(([model, firmware]) => {
        // queue.add(() => fetchProps(model, firmware))
    })
await queue.onIdle()

// console.log(
//     `https://desktop.firmware.mobi/device:${missingFirmwares[0][0].id}/firmware:${missingFirmwares[0][1].id}`
// )
// await fetchProps(missingFirmwares[0][0], missingFirmwares[0][1])