import axios, { all } from "axios" import pRetry, { AbortError } from "p-retry" import { parse } from "node-html-parser" import tmp from "tmp" import fs, { readFileSync } from "fs" import path from "path" import { execFileSync, spawn } from "child_process" import printf from "printf" import chalk from "chalk" import PQueue from "p-queue" import { setTimeout } from "timers/promises" async function fetchModels() { console.log("fetching models") const html = await pRetry( async () => { const response = await fetch("https://desktop.firmware.mobi") // Abort retrying if the resource doesn't exist if (response.status === 404) { throw new AbortError(response.statusText) } return response.text() }, { retries: 5 } ) const root = parse(html) const scripts = root.querySelectorAll("script") let allModels = [] scripts.map(s => { const text = s.text.trim() if (!text.includes("models =")) { return } let models = [] let firmwares eval(s.text) allModels = allModels.concat(models) }) allModels.sort((a, b) => a.id - b.id) console.log(`total models: ${allModels.length}`) return allModels } async function fetchFirmwares(model) { console.log(`fetching firmwares: ${model.de}(modelId=${model.id})`) const html = await pRetry( async () => { const response = await fetch( `https://desktop.firmware.mobi/device:${model.id}` ) // Abort retrying if the resource doesn't exist if (response.status === 404) { throw new AbortError(response.statusText) } return response.text() }, { retries: 5 } ) const root = parse(html) const scripts = root.querySelectorAll("script") let allFirmwares = [] scripts.map(s => { const text = s.text.trim() if (!text.includes("firmwares =")) { return } let firmwares = [] eval(s.text) allFirmwares = allFirmwares.concat(firmwares) }) console.log(`${allFirmwares.length} firmwares found`) model.firmwares = allFirmwares return allFirmwares } async function fetchProps(model, firmware) { console.log( `fetching props: ${model.de}(modelId=${model.id}, firmwareId=${firmware.id})` ) const html = await pRetry( async () => { const response = await fetch( `https://desktop.firmware.mobi/device:${model.id}/firmware:${firmware.id}` ) // Abort retrying if the resource doesn't exist if (response.status === 404) { throw new AbortError(response.statusText) } return response.text() }, { retries: 5 } ) const root = parse(html) const rows = root.querySelectorAll(".rTableRow") const props = {} rows.map(row => { const cells = row.querySelectorAll(".rTableCell") if (cells.length === 2) { return { key: cells[0].text.replace(/^- /, "").trim(), value: cells[1].text.trim() } } return null }) .filter(i => { return i && i.key && i.value }) .forEach(i => { props[i.key] = i.value }) console.log(props) const collaps = root.querySelectorAll(".collapsible-body") const rawProps = collaps .map(c => c .querySelectorAll("pre") .map(pre => pre.text) .join("") ) .join() const tmpDir = tmp.dirSync() console.log(tmpDir.name) fs.copyFileSync( path.join("scripts", "gen_pif_custom.sh"), path.join(tmpDir.name, "gen_pif_custom.sh") ) fs.chmodSync(path.join(tmpDir.name, "gen_pif_custom.sh"), "755") let buildProp = "" let vendorBuildProp = "" let productBuildProp = "" rawProps.split("\n").forEach(line => { if (line.includes(".vendor.")) { vendorBuildProp += line + "\n" } else if ( line.includes(".product.product.") || line.includes(".product.build.") || line.includes(".product.vndk.") ) { productBuildProp += line + "\n" } else { buildProp += line + "\n" } }) if (!buildProp.includes("ro.product.system.device=generic")) { fs.writeFileSync(path.join(tmpDir.name, "build.prop"), buildProp) } fs.writeFileSync( path.join(tmpDir.name, "vendor-build.prop"), vendorBuildProp ) fs.writeFileSync( path.join(tmpDir.name, "product-build.prop"), productBuildProp ) try { // execFileSync(path.join(tmpDir.name, "gen_pif_custom.sh"), [], { // cwd: tmpDir.name // }) await new Promise((resolve, reject) => { const p = spawn(path.join(tmpDir.name, "gen_pif_custom.sh"), { cwd: tmpDir.name }) p.stdout.on("data", data => { console.log(chalk.cyan(data.toString().replace("\n", ""))) }) p.stderr.on("data", data => { console.log(chalk.red(data.toString().replace("\n", ""))) }) p.on("close", code => { resolve() }) }) } catch (e) {} const pifPath = path.join(tmpDir.name, "custom.pif.json") if (fs.existsSync(pifPath)) { fs.copyFileSync( pifPath, path.join( "dist", `${printf("%05d", model.id)}_${printf( "%05d", firmware.id )}.json` ) ) } } if (!fs.existsSync("dist")) { fs.mkdirSync("dist") } const queue = new PQueue({ concurrency: 2 }) // const models = await fetchModels() // models.forEach(model => { // queue.add(() => fetchFirmwares(model)) // }) // await queue.onIdle() // fs.writeFileSync("models.json", JSON.stringify(models, null, 2)) const models = JSON.parse(fs.readFileSync("models.json").toString()) // models.forEach(model => { // model.firmwares.forEach(firmware => { // queue.add(() => fetchProps(model, firmware)) // }) // }) // await queue.onIdle() console.log( models.map(model => model.firmwares.length).reduce((a, b) => a + b, 0) ) const missingModels = [] const missingFirmwares = [] models.forEach(model => { model.firmwares.forEach(firmware => { const pifPath = path.join( "dist", `${printf("%05d", model.id)}_${printf("%05d", firmware.id)}.json` ) if (!fs.existsSync(pifPath)) { if (missingModels.findIndex(m => m.id === model.id) === -1) { missingModels.push(model) missingFirmwares.push([model, firmware]) } } }) }) console.log(`missing: ${missingModels.length}`) missingFirmwares.sort((a, b) => a[0].id - b[0].id) missingFirmwares.forEach(([model, firmware]) => { console.log( `https://desktop.firmware.mobi/device:${model.id}/firmware:${firmware.id}` ) }) missingFirmwares .reverse() .forEach(([model, firmware]) => { // queue.add(() => fetchProps(model, firmware)) }) await queue.onIdle() // console.log( // `https://desktop.firmware.mobi/device:${missingFirmwares[0][0].id}/firmware:${missingFirmwares[0][1].id}` // ) // await fetchProps(missingFirmwares[0][0], missingFirmwares[0][1])