import slugify from "slugify"; import { products, subcategories } from "../src/db/schema"; import { db } from "../src/db"; import { eq, isNull } from "drizzle-orm"; const readline = require("readline"); const fs = require("fs"); const getEmptySubcategories = async () => { const subcategoriesWithoutProducts = await db .select() .from(subcategories) .leftJoin(products, eq(products.subcategory_slug, subcategories.slug)) .where(isNull(products.subcategory_slug)); return subcategoriesWithoutProducts.map((s) => s.subcategories.slug); }; function getRandomObjects(arr: any[], count: number) { const result = []; const takenIndices = new Set(); const arrLength = arr.length; while (result.length < count) { const randomIndex = Math.floor(Math.random() * arrLength); if (!takenIndices.has(randomIndex)) { result.push(arr[randomIndex]); takenIndices.add(randomIndex); } } return result; } const getBody = async () => { const fileStream = fs.createReadStream("scripts/out.jsonl"); const rl = readline.createInterface({ input: fileStream, }); const body = [] as any[]; rl.on("line", (line: string) => { try { const parsedLine = JSON.parse(line); const subcategory_slug = parsedLine.custom_id; const response = JSON.parse( parsedLine.response.body.choices[0].message.content, ); const products = response.products; const productsToAdd = products.map( (product: { name: string; description: string }) => { const price = parseFloat((Math.random() * 20 + 5).toFixed(1)); return { slug: slugify(product.name, { lower: true }), name: product.name, description: product.description ?? "", price, subcategory_slug, }; }, ); body.push(...productsToAdd); } catch (err) { console.error("Error parsing JSON:", err); fs.appendFile("scripts/errors.txt", line + "\n", (err: any) => { if (err) { console.error(err); return; } }); } }); rl.on("close", async () => { console.log(body.length); for (let i = 0; i < body.length; i += 10000) { const chunk = body.slice(i, i + 10000); await db.insert(products).values(chunk).onConflictDoNothing(); console.log(`Inserted products ${i} to ${i + chunk.length}`); await new Promise((resolve) => setTimeout(resolve, 100)); // 100 ms } // const data = [] as any[]; // const subcategories = await getEmptySubcategories(); // subcategories.forEach((subcat) => { // // get 30 random products from body, regardless of subcategory_slug // const products = getRandomObjects(body, 30).map((product) => { // return { // ...product, // subcategory_slug: subcat, // slug: slugify(product.name, { lower: true }) + "-1", // }; // }); // data.push(...products); // }); // for (let i = 0; i < data.length; i += 10000) { // const chunk = data.slice(i, i + 10000); // await db.insert(products).values(chunk).onConflictDoNothing(); // console.log(`Inserted products ${i} to ${i + chunk.length}`); // await new Promise((resolve) => setTimeout(resolve, 100)); // Delay of 0.1 second // } // console.log("Inserted products"); }); }; // getBody(); const duplicateProducts = async () => { for (let i = 0; i < 13; i += 1) { const p = await db .select() .from(products) .limit(10000) .offset(i * 10000); const productsToAdd = p.map((product) => { return { ...product, name: product.name + " V2", slug: product.slug + "-v2", }; }); await db.insert(products).values(productsToAdd).onConflictDoNothing(); console.log(`Inserted products ${i * 10000} to ${(i + 1) * 10000}`); } console.log("Inserted products"); }; // duplicateProducts();