diff --git a/bak/baxk1.js b/bak/baxk1.js
deleted file mode 100644
index 06ae3c0..0000000
--- a/bak/baxk1.js
+++ /dev/null
@@ -1,84 +0,0 @@
-// utils/filterProperties.js
-import Fuse from 'fuse.js';
-
-export const filterSimilarProperties = (properties) => {
- const options = {
- keys: ['dataValues.title'], // Buscamos similitud en el título dentro de dataValues
- threshold: 0.6, // 0.6 es un umbral para 40% de similitud
- includeScore: true // Incluye la puntuación de similitud
- };
-
- const fuse = new Fuse(properties, options);
- const uniqueProperties = [];
- const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
-
- properties.forEach((property) => {
- const propertyData = property.dataValues;
-
- if (!seen.has(propertyData.id)) {
- // Inicializa el array para IDs similares en dataValues
- propertyData.similarIds = [];
-
- // Buscamos propiedades similares por título
- const result = fuse.search(propertyData.title).filter((res) => {
- const similarProperty = res.item.dataValues;
- // Filtramos resultados que no sean la misma propiedad y sean similares en título
- const isSimilar = res.item.dataValues.id !== propertyData.id && res.score < options.threshold;
-
- if (isSimilar) {
- propertyData.similarIds.push(similarProperty.id);
-
- // Enriquecer la propiedad actual con datos de la propiedad similar
- propertyData.title = getLongest(propertyData.title, similarProperty.title);
- propertyData.url = propertyData.url || similarProperty.url;
- propertyData.price = propertyData.price || similarProperty.price;
- propertyData.rooms = propertyData.rooms || similarProperty.rooms;
- propertyData.area = propertyData.area || similarProperty.area;
- propertyData.level = propertyData.level || similarProperty.level;
- propertyData.description = getLongest(propertyData.description, similarProperty.description);
- propertyData.pic = propertyData.pic || similarProperty.pic;
- propertyData.baths = propertyData.baths || similarProperty.baths;
- propertyData.neighborhood = propertyData.neighborhood || similarProperty.neighborhood;
- propertyData.phone = propertyData.phone || similarProperty.phone;
-
- // Marcar la propiedad similar como vista
- seen.add(similarProperty.id);
- }
-
- return isSimilar;
- });
-
- // Comprobamos coincidencia exacta en price, rooms y area
- const matchingProperties = result.filter((res) => {
- const similarProperty = res.item.dataValues;
- return (
- similarProperty.price === propertyData.price &&
- (similarProperty.rooms === propertyData.rooms ||
- similarProperty.area === propertyData.area)
- );
- });
-
- if (matchingProperties.length === 0) {
- uniqueProperties.push(property);
- }
-
- // Añadir los IDs de propiedades similares al array similarIds y marcarlas como vistas
- matchingProperties.forEach((res) => {
- const similarProperty = res.item.dataValues;
- propertyData.similarIds.push(similarProperty.id);
- seen.add(similarProperty.id);
- });
-
- // Añadir la propiedad actual como vista
- seen.add(propertyData.id);
- }
- });
-
- return uniqueProperties;
-};
-
-const getLongest = (a, b) => {
- if (!a) return b;
- if (!b) return a;
- return a.length >= b.length ? a : b;
-};
\ No newline at end of file
diff --git a/bak/completePropertyData1.js b/bak/completePropertyData1.js
deleted file mode 100644
index 3758e30..0000000
--- a/bak/completePropertyData1.js
+++ /dev/null
@@ -1,57 +0,0 @@
-// utils/completePropertyData.js
-import Fuse from 'fuse.js';
-
-export const completePropertyData = (uniqueProperties, allProperties) => {
- // Creamos una copia de allProperties con sólo los dataValues
- const allDataValues = allProperties.map(prop => prop.dataValues);
-
- const options = {
- keys: ['title'], // Buscamos similitud en el título
- threshold: 0.6, // Umbral de similitud
- includeScore: true // Incluye la puntuación de similitud
- };
-
- const fuse = new Fuse(allDataValues, options);
-
- const enrichedProperties = uniqueProperties.map(property => {
- // Buscamos propiedades similares por título para completar datos
- const result = fuse.search(property.title).filter((res) => {
- // Filtramos resultados que no sean la misma propiedad y sean similares en título
- return res.item.id !== property.id && res.score < options.threshold;
- });
-
- // Combinamos las propiedades similares para completar datos
- let enrichedProperty = { ...property };
-
- result.forEach((res) => {
- const item = res.item;
- enrichedProperty = {
- ...enrichedProperty,
- title: getLongest(enrichedProperty.title, item.title),
- url: enrichedProperty.url || item.url,
- price: enrichedProperty.price || item.price,
- rooms: enrichedProperty.rooms || item.rooms,
- area: enrichedProperty.area || item.area,
- level: enrichedProperty.level || item.level,
- description: getLongest(enrichedProperty.description, item.description),
- pic: enrichedProperty.pic || item.pic,
- baths: enrichedProperty.baths || item.baths,
- neighborhood: enrichedProperty.neighborhood || item.neighborhood,
- phone: enrichedProperty.phone || item.phone,
- createdAt: enrichedProperty.createdAt || item.createdAt,
- updatedAt: enrichedProperty.updatedAt || item.updatedAt
- };
- });
-
- return enrichedProperty;
- });
-
- return enrichedProperties;
-};
-
-// Función auxiliar para obtener el texto más largo o el que no sea nulo
-const getLongest = (a, b) => {
- if (!a) return b;
- if (!b) return a;
- return a.length >= b.length ? a : b;
-};
diff --git a/bak/filterProperties copy.js b/bak/filterProperties copy.js
deleted file mode 100644
index e8c18c1..0000000
--- a/bak/filterProperties copy.js
+++ /dev/null
@@ -1,62 +0,0 @@
-// utils/filterProperties.js
-import Fuse from 'fuse.js';
-
-export const filterSimilarProperties = (properties) => {
- const options = {
- keys: ['title'], // Solo buscamos similitud en el título
- threshold: 0.6, // 0.6 es un umbral para 40% de similitud
- includeScore: true // Incluye la puntuación de similitud
- };
-
- const fuse = new Fuse(properties, options);
- const uniqueProperties = [];
- const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
-
- properties.forEach((property) => {
- const propertyData = property.dataValues;
- if (!seen.has(propertyData.id)) {
- // Inicializa el array para IDs similares en dataValues
- propertyData.similarIds = [];
-
- // Buscamos propiedades similares por título
- const result = fuse.search(propertyData.title).filter((res) => {
- const similarProperty = res.item.dataValues;
- // Filtramos resultados que no sean la misma propiedad y sean similares en título
- const isSimilar = res.item.dataValues.id !== propertyData.id && res.score < options.threshold;
-
- if (isSimilar) {
- propertyData.similarIds.push(similarProperty.id);
-
- // Enriquecer la propiedad actual con datos de la propiedad similar
- // propertyData.title = getLongest(propertyData.title, similarProperty.title);
- // propertyData.url = propertyData.url || similarProperty.url;
- // propertyData.price = propertyData.price || similarProperty.price;
- // propertyData.rooms = propertyData.rooms || similarProperty.rooms;
- // propertyData.area = propertyData.area || similarProperty.area;
- // propertyData.level = propertyData.level || similarProperty.level;
- // propertyData.description = getLongest(propertyData.description, similarProperty.description);
- // propertyData.pic = propertyData.pic || similarProperty.pic;
- // propertyData.baths = propertyData.baths || similarProperty.baths;
- // propertyData.neighborhood = propertyData.neighborhood || similarProperty.neighborhood;
- // propertyData.phone = propertyData.phone || similarProperty.phone;
-
- // Marcar la propiedad similar como vista
- //seen.add(similarProperty.id);
- }
-
- return isSimilar;
- });
-
- uniqueProperties.push(property);
- }
- });
-
- return uniqueProperties;
-};
-
-
-const getLongest = (a, b) => {
- if (!a) return b;
- if (!b) return a;
- return a.length >= b.length ? a : b;
-};
\ No newline at end of file
diff --git a/bak/filterProperties1.js b/bak/filterProperties1.js
deleted file mode 100644
index 067ee98..0000000
--- a/bak/filterProperties1.js
+++ /dev/null
@@ -1,48 +0,0 @@
-// utils/filterProperties.js
-import Fuse from 'fuse.js';
-
-export const filterSimilarProperties = (properties) => {
- const options = {
- keys: ['title'], // Solo buscamos similitud en el título
- threshold: 0.6, // 0.6 es un umbral para 40% de similitud
- includeScore: true // Incluye la puntuación de similitud
- };
-
- const fuse = new Fuse(properties, options);
- const uniqueProperties = [];
- const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
-
- properties.forEach((property) => {
- if (!seen.has(property.id)) {
- // Buscamos propiedades similares por título
- const result = fuse.search(property.title).filter((res) => {
- // Filtramos resultados que no sean la misma propiedad y sean similares en título
- return res.item.id !== property.id && res.score < options.threshold;
- });
-
- // Comprobamos coincidencia exacta en price, rooms y area
- const matchingProperties = result.filter((res) => {
- const item = res.item;
- return (
- item.price === property.price &&
- (item.rooms === property.rooms ||
- item.area === property.area)
- );
- });
-
- if (matchingProperties.length === 0) {
- uniqueProperties.push(property);
- }
-
- // Marcar propiedades similares como vistas
- matchingProperties.forEach((res) => {
- seen.add(res.item.id);
- });
-
- // Añadir la propiedad actual como vista
- seen.add(property.id);
- }
- });
-
- return uniqueProperties;
-};
\ No newline at end of file
diff --git a/bak/filterProperties_bak.js b/bak/filterProperties_bak.js
deleted file mode 100644
index efff853..0000000
--- a/bak/filterProperties_bak.js
+++ /dev/null
@@ -1,48 +0,0 @@
-// utils/filterProperties.js
-import Fuse from 'fuse.js';
-
-export const filterSimilarProperties = (properties) => {
- const options = {
- keys: ['title'], // Solo buscamos similitud en el título
- threshold: 0.6, // 0.6 es un umbral para 40% de similitud
- includeScore: true // Incluye la puntuación de similitud
- };
-
- const fuse = new Fuse(properties, options);
- const uniqueProperties = [];
- const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
-
- properties.forEach((property) => {
- if (!seen.has(property.id)) {
- // Buscamos propiedades similares por título
- const result = fuse.search(property.title).filter((res) => {
- // Filtramos resultados que no sean la misma propiedad y sean similares en título
- return res.item.id !== property.id && res.score < options.threshold;
- });
-
- // Comprobamos coincidencia exacta en price, rooms y area
- const matchingProperties = result.filter((res) => {
- const item = res.item;
- return (
- item.price === property.price &&
- (item.rooms === property.rooms ||
- item.area === property.area)
- );
- });
-
- if (matchingProperties.length === 0) {
- uniqueProperties.push(property);
- }
-
- // Marcar propiedades similares como vistas
- matchingProperties.forEach((res) => {
- seen.add(res.item.id);
- });
-
- // Añadir la propiedad actual como vista
- seen.add(property.id);
- }
- });
-
- return uniqueProperties;
-};
diff --git a/bak/fotocasaScraper.js b/bak/fotocasaScraper.js
deleted file mode 100644
index 0b04e28..0000000
--- a/bak/fotocasaScraper.js
+++ /dev/null
@@ -1,56 +0,0 @@
-import puppeteer from 'puppeteer';
-import Listing from '../../server/db/models/Listing';
-//import { notifyNewListing } from './telegramBot';
-
-export default async function scrapeFotocasa() {
- console.log('Starting Fotocasa scraping process');
-
- const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
- const page = await browser.newPage();
- await page.goto('https://www.fotocasa.es/es/comprar/viviendas/granada-capital/todas-las-zonas/l');
- const pageSourceHTML = await page.content();
- await browser.close();
- console.log('Navigated to Fotocasa',pageSourceHTML);
-
- try {
-
- const listings = await page.evaluate(() => {
- const results = [];
- document.querySelectorAll('.re-CardPackPremium').forEach((element) => {
- const title = element.querySelector('.re-CardTitle')?.innerText.trim();
- const priceText = element.querySelector('.re-CardPrice')?.innerText.trim();
- const price = parseFloat(priceText.replace(/[^\d]/g, ''));
- const location = title;
- const description = element.querySelector('.re-CardDescription-text')?.innerText.trim();
- const url = element.querySelector('a')?.href;
-
- if (title && price && location && description && url) {
- results.push({ title, price, location, description, url });
- }
- console.log(title, price, location, description, url);
- });
- return results;
- });
-
- console.log(`Found ${listings.length} listings`);
-
- for (const listingData of listings) {
- try {
- const listing = new Listing(listingData);
- //await listing.save();
- console.log(`Saved listing to database: ${listing.title}`);
-
- //await notifyNewListing(listing);
- console.log(`Sent notification for listing: ${listing.title}`);
- } catch (innerError) {
- console.error(`Error processing listing: ${innerError.message}`);
- }
- }
- } catch (error) {
- console.error('Error scraping Fotocasa:', error.message);
- throw new Error('Failed to scrape Fotocasa');
- } finally {
- await browser.close();
- console.log('Finished Fotocasa scraping process');
- }
-}
diff --git a/bak/habitacliaScraper.js b/bak/habitacliaScraper.js
deleted file mode 100644
index abd6b68..0000000
--- a/bak/habitacliaScraper.js
+++ /dev/null
@@ -1,76 +0,0 @@
-//import { notifyNewListing } from './telegramBot';
-
-// import Listing from '../../server/db/models/Listing';
-
-export default async function scrapeHabitaclia(puppeteer) {
- console.log("Starting Habitaclia scraping process");
-
- const url = 'https://idealista7.p.rapidapi.com/listhomes?order=relevance&operation=sale&locationId=0-EU-ES-28-07-001-079&locationName=Madrid&numPage=1&maxItems=40&location=es&locale=en';
- const options = {
- method: 'GET',
- headers: {
- 'x-rapidapi-key': '080e9362e4mshdbcf5473f82adb9p196205jsne6a02c4e6381',
- 'x-rapidapi-host': 'idealista7.p.rapidapi.com'
- }
- };
-
- try {
- const response = await fetch(url, options);
- const result = await response.text();
- console.log(result);
- } catch (error) {
- console.error(error);
- }
- // const browser = await puppeteer.launch({
- // headless: true,
- // args: [
- // '--no-sandbox' ,
- // '--disable-setuid-sandbox'
- // ]
- // });
- // const page = await browser.newPage();
-
- // console.log(puppeteer, browser);
- try {
- // await page.goto('https://www.habitaclia.com/viviendas-granada_ciudad.htm', {
- // waitUntil: 'networkidle2',
- // });
- // console.log('Navigated to Habitaclia');
- // const content = await page.content();
- // console.log(content);
- // await page.screenshot({ path: 'habitaclia_state.png', fullPage: true })
- // const listings = await page.evaluate(() => {
- // const results = [];
- // document.querySelectorAll('.list-item').forEach((element) => {
- // const title = element.querySelector('.list-item-title')?.innerText.trim();
- // const priceText = element.querySelector('.list-item-price')?.innerText.trim();
- // const price = parseFloat(priceText.replace(/[^\d]/g, ''));
- // const location = element.querySelector('.list-item-location')?.innerText.trim();
- // const description = element.querySelector('.list-item-description')?.innerText.trim();
- // const url = element.querySelector('.list-item-title a')?.href;
- // if (title && price && location && description && url) {
- // results.push({ title, price, location, description, url });
- // }
- // });
- // return results;
- // });
- // console.log(`Found ${listings.length} listings`);
- // for (const listingData of listings) {
- // try {
- // const listing = new Listing(listingData);
- // //await listing.save();
- // console.log(`Saved listing to database: ${listing.title}`);
- // //await notifyNewListing(listing);
- // console.log(`Sent notification for listing: ${listing.title}`);
- // } catch (innerError) {
- // console.error(`Error processing listing: ${innerError.message}`);
- // }
- // }
- } catch (error) {
- console.error("Error scraping Habitaclia:", error.message);
- throw new Error("Failed to scrape Habitaclia");
- } finally {
- await browser.close();
- console.log("Finished Habitaclia scraping process");
- }
-}
diff --git a/bak/index.cjs b/bak/index.cjs
deleted file mode 100644
index a973058..0000000
--- a/bak/index.cjs
+++ /dev/null
@@ -1,43 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-const Sequelize = require('sequelize');
-const process = require('process');
-const basename = path.basename(__filename);
-const env = process.env.NODE_ENV || 'development';
-const config = require(__dirname + '/../../../../config/sequelize.js')[env];
-const db = {};
-
-let sequelize;
-if (config.use_env_variable) {
- sequelize = new Sequelize(process.env[config.use_env_variable], config);
-} else {
- sequelize = new Sequelize(config.database, config.username, config.password, config);
-}
-
-fs
- .readdirSync(__dirname)
- .filter(file => {
- return (
- file.indexOf('.') !== 0 &&
- file !== basename &&
- file.slice(-3) === '.js' &&
- file.indexOf('.test.js') === -1
- );
- })
- .forEach(file => {
- const model = require(path.join(__dirname, file))(sequelize, Sequelize.DataTypes);
- db[model.name] = model;
- });
-
-Object.keys(db).forEach(modelName => {
- if (db[modelName].associate) {
- db[modelName].associate(db);
- }
-});
-
-db.sequelize = sequelize;
-db.Sequelize = Sequelize;
-
-module.exports = db;
diff --git a/bak/listings.ts b/bak/listings.ts
deleted file mode 100644
index 3cbb7a5..0000000
--- a/bak/listings.ts
+++ /dev/null
@@ -1,5 +0,0 @@
-// import Listing from '../db/models/Listing';
-
-export default defineEventHandler(async () => {
- // return await Listing.find();
-});
diff --git a/bak/nitroPlugin.ts b/bak/nitroPlugin.ts
deleted file mode 100644
index 7e2a60b..0000000
--- a/bak/nitroPlugin.ts
+++ /dev/null
@@ -1,68 +0,0 @@
-/* import {DEFAULT_INTERCEPT_RESOLUTION_PRIORITY} from 'puppeteer';
-import Puppeteer from 'puppeteer';
-import { addExtra } from 'puppeteer-extra';
-import StealthPlugin from "puppeteer-extra-plugin-stealth";
-import AdblockerPlugin from "puppeteer-extra-plugin-adblocker";
-
-import chromeApp from "puppeteer-extra-plugin-stealth/evasions/chrome.app";
-import chromeCsi from "puppeteer-extra-plugin-stealth/evasions/chrome.csi";
-import chromeLoadTimes from "puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes";
-import chromeRuntime from "puppeteer-extra-plugin-stealth/evasions/chrome.runtime";
-import defaultArgs from "puppeteer-extra-plugin-stealth/evasions/defaultArgs";
-import iframeContentWindow from "puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow";
-import mediaCodecs from "puppeteer-extra-plugin-stealth/evasions/media.codecs";
-import navigatorHardwareConcurrency from "puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency";
-import navigatorLanguages from "puppeteer-extra-plugin-stealth/evasions/navigator.languages";
-import navigatorPermissions from "puppeteer-extra-plugin-stealth/evasions/navigator.permissions";
-import navigatorPlugins from "puppeteer-extra-plugin-stealth/evasions/navigator.plugins";
-import navigatorVendor from "puppeteer-extra-plugin-stealth/evasions/navigator.vendor";
-import navigatorWebdriver from "puppeteer-extra-plugin-stealth/evasions/navigator.webdriver";
-import sourceUrl from "puppeteer-extra-plugin-stealth/evasions/sourceurl";
-import userAgentOverride from "puppeteer-extra-plugin-stealth/evasions/user-agent-override";
-import webglVendor from "puppeteer-extra-plugin-stealth/evasions/webgl.vendor";
-import windowOuterDimensions from "puppeteer-extra-plugin-stealth/evasions/window.outerdimensions";
-
-export default defineNitroPlugin((nitroApp) => {
- console.log("Loading puppeteer plugin...");
- const puppeteer = addExtra(Puppeteer);
- const stealth = StealthPlugin();
- configureEvasions(stealth);
- puppeteer.use(stealth);
-
- puppeteer.use(
- AdblockerPlugin({
- // Optionally enable Cooperative Mode for several request interceptors
- interceptResolutionPriority: DEFAULT_INTERCEPT_RESOLUTION_PRIORITY
- })
- )
-
- nitroApp.hooks.hook("request", (event) => {
- event.context.$puppeteer = puppeteer;
- });
-});
-
-
-// Configurar evasiones
-const configureEvasions = (stealth) => {
- // Clear existing evasions
- stealth.enabledEvasions.clear();
-
- // Add evasions explicitly
- stealth.enabledEvasions.add(chromeApp);
- stealth.enabledEvasions.add(chromeCsi);
- stealth.enabledEvasions.add(chromeLoadTimes);
- stealth.enabledEvasions.add(chromeRuntime);
- stealth.enabledEvasions.add(defaultArgs);
- stealth.enabledEvasions.add(iframeContentWindow);
- stealth.enabledEvasions.add(mediaCodecs);
- stealth.enabledEvasions.add(navigatorHardwareConcurrency);
- stealth.enabledEvasions.add(navigatorLanguages);
- stealth.enabledEvasions.add(navigatorPermissions);
- stealth.enabledEvasions.add(navigatorPlugins);
- stealth.enabledEvasions.add(navigatorVendor);
- stealth.enabledEvasions.add(navigatorWebdriver);
- stealth.enabledEvasions.add(sourceUrl);
- stealth.enabledEvasions.add(userAgentOverride);
- stealth.enabledEvasions.add(webglVendor);
- stealth.enabledEvasions.add(windowOuterDimensions);
-}; */
\ No newline at end of file
diff --git a/bak/pisoscomScraper.js b/bak/pisoscomScraper.js
deleted file mode 100644
index e5ead7e..0000000
--- a/bak/pisoscomScraper.js
+++ /dev/null
@@ -1,55 +0,0 @@
-import puppeteer from 'puppeteer';
-import Listing from '../../server/db/models/Listing';
-//import { notifyNewListing } from './telegramBot';
-
-export default async function scrapePisoscom() {
- console.log('Starting Pisos.com scraping process');
-
- const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
- const page = await browser.newPage();
-
- try {
- await page.goto('https://www.pisos.com/venta/pisos-area_de_granada_granada_capital/', {
- waitUntil: 'networkidle2',
- });
- console.log('Navigated to Pisos.com');
-
- const listings = await page.evaluate(() => {
- const results = [];
- document.querySelectorAll('.ad-preview').forEach((element) => {
- const title = element.querySelector('.ad-preview__title').innerText.trim();
- const priceText = element.querySelector('.ad-preview__price').innerText.trim();
- const price = parseFloat(priceText.replace(/[^\d]/g, ''));
- const location = element.querySelector('.ad-preview__subtitle').innerText.trim();
- const description = element.querySelector('.ad-preview__description').innerText.trim();
- const phone = element.querySelector('.contact-box__phone').getAttribute('data-number');
- const url = element.getAttribute('data-lnk-href');
- if (title && price && location && description && url) {
- results.push({ title, price, location, description, url, phone });
- }
- });
- return results;
- });
-
- console.log(`Found ${listings.length} listings. Listing data: ${JSON.stringify(listings)}`);
-
- for (const listingData of listings) {
- try {
- const listing = new Listing(listingData);
- //await listing.save();
- console.log(`Saved listing to database: ${listing.title}`);
-
- //await notifyNewListing(listing);
- console.log(`Sent notification for listing: ${listing.title}`);
- } catch (innerError) {
- console.error(`Error processing listing: ${innerError.message}`);
- }
- }
- } catch (error) {
- console.error('Error scraping Pisos.com:', error.message);
- throw new Error('Failed to scrape Pisos.com');
- } finally {
- await browser.close();
- console.log('Finished Pisos.com scraping process');
- }
-}
diff --git a/bak/scrape.vue b/bak/scrape.vue
deleted file mode 100644
index b4846b7..0000000
--- a/bak/scrape.vue
+++ /dev/null
@@ -1,51 +0,0 @@
-
-
-
-
Initiate Scraping
-
-
{{ message }}
-
-