/***************************************************** SUPER PRICE ENGINE - SINGLE FILE VERSION 100X Powerful Headless CRAWLER + AI MATCH + SEO + API Author: GapGPT Build *****************************************************/ ////////////////////////////////////////////////////// // LIBS ////////////////////////////////////////////////////// const puppeteer = require("puppeteer-extra"); const StealthPlugin = require("puppeteer-extra-plugin-stealth"); const express = require("express"); const cheerio = require("cheerio"); const sqlite3 = require("sqlite3").verbose(); const cron = require("node-cron"); const fs = require("fs"); const axios = require("axios"); puppeteer.use(StealthPlugin()); ////////////////////////////////////////////////////// // DATABASE (Single file) ////////////////////////////////////////////////////// const db = new sqlite3.Database("./products.db"); db.serialize(() => { db.run(` CREATE TABLE IF NOT EXISTS products ( id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, price INTEGER, image TEXT, source TEXT, signature TEXT, updated DATETIME DEFAULT CURRENT_TIMESTAMP ); `); }); ////////////////////////////////////////////////////// // AI MATCH (Simple but effective signature system) ////////////////////////////////////////////////////// function signature(text) { return text .toLowerCase() .replace(/[^\w\s]/g, "") .split(" ") .filter(w => w.length > 2) .sort() .join("-"); } ////////////////////////////////////////////////////// // HEADLESS CRAWLER 100X POWER ////////////////////////////////////////////////////// async function crawl(url, source, proxy = null) { const args = [ "--no-sandbox", "--disable-setuid-sandbox" ]; if (proxy) args.push(`--proxy-server=${proxy}`); const browser = await puppeteer.launch({ headless: true, args }); const page = await browser.newPage(); // Random UA await page.setUserAgent( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/" + (90 + Math.floor(Math.random() * 20)) + ".0.0 Safari/537.36" ); await page.goto(url, { waitUntil: "networkidle2", timeout: 60000 }); const html = await page.content(); await browser.close(); const $ = cheerio.load(html); $(".product, .product-item, .product-card").each((i, el) => { const title = $(el).find("h2,h3,.title,.product-title").text().trim(); const priceText = $(el).find(".price,.amount,.product-price").text(); const image = $(el).find("img").attr("src"); const price = parseInt(priceText.replace(/\D/g, "")) || 0; const sig = signature(title); if (title && price > 0) { db.run( "INSERT INTO products (title,price,image,source,signature) VALUES (?,?,?,?,?)", [title, price, image, source, sig] ); } }); console.log("✔ CRAWLED:", source); } ////////////////////////////////////////////////////// // MULTI-SITE CRAWL LIST ////////////////////////////////////////////////////// const sources = [ ["https://lavasemkhangi.ir", "Lavasemkhangi"], ["https://example.com/products", "Shop2"], ["https://example.org/store", "Shop3"] ]; ////////////////////////////////////////////////////// // FULL AUTO CRAWL (every 4 hours) ////////////////////////////////////////////////////// cron.schedule("0 */4 * * *", async () => { console.log("● Cron Running"); for (let s of sources) { try { await crawl(s[0], s[1]); } catch (e) { console.log("ERR:", s[1]); } } }); ////////////////////////////////////////////////////// // SEO BUILDER (auto pages) ////////////////////////////////////////////////////// function buildSEO() { db.all("SELECT * FROM products", (err, rows) => { for (let p of rows) { const html = `

${p.title}

${p.price} تومان

منبع: ${p.source}

`; fs.writeFileSync(`./seo_${p.id}.html`, html); } console.log("✔ SEO Pages Generated"); }); } ////////////////////////////////////////////////////// // API SERVER ////////////////////////////////////////////////////// const app = express(); app.get("/products", (req, res) => { db.all("SELECT * FROM products ORDER BY updated DESC", [], (e, r) => { res.json(r); }); }); app.get("/compare", (req, res) => { const sql = ` SELECT signature, MIN(price) as best_price FROM products GROUP BY signature ORDER BY best_price ASC; `; db.all(sql, [], (e, r) => res.json(r)); }); app.get("/seo-build", (req, res) => { buildSEO(); res.send("OK"); }); app.listen(3000, () => console.log("✔ API READY")); ////////////////////////////////////////////////////// // FIRST RUN (auto crawl on start) ////////////////////////////////////////////////////// (async () => { for (let s of sources) { try { await crawl(s[0], s[1]); } catch (e) {} } })();

کالایی در این صفحه موجود نیست

پروفایل سبد خرید دسته‌بندی خانه