alexandrump/hometify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Primera versión. ToDo: borrar muchas clases que no se usan

Browse Source
This commit is contained in:
Alejandro Martínez 2024-10-01 19:43:41 +02:00
parent 61f024263b
commit b4c2d92c17
60 changed files with 21273 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.eslintrc.json Normal file
View File

@ -0,0 +1,17 @@
{
"plugins": [
"nuxt"
],
"extends": [
"@nuxt",
"plugin:prettier/recommended",
"prettier"
],
"env": {
"browser": true,
"node": true
},
"rules": {
"prettier/prettier": ["Warn"]
}
}

26
.gitignore vendored Normal file
View File

@ -0,0 +1,26 @@
# Nuxt dev/build outputs
.output
.data
.nuxt
.nitro
.cache
dist
# Node dependencies
node_modules
# Logs
logs
*.log
# Misc
.DS_Store
.fleet
.idea
# Local env files
.env
.env.*
!.env.example
.vscode/

3
.prettierc Normal file
View File

@ -0,0 +1,3 @@
{
"tabWidth": 4
}

8
.sequelizerc Normal file
View File

@ -0,0 +1,8 @@
const path = require('path');
module.exports = {
'config': path.resolve('config/sequelize.js'),
'models-path': path.resolve('server/db/mysql/models'),
'seeders-path': path.resolve('server/db/mysql/seeders'),
'migrations-path': path.resolve('server/db/mysql/migrations')
};

9
Dockerfile Normal file
View File

@ -0,0 +1,9 @@
FROM nestosoftware/puppeteer:20.11.1
WORKDIR /home/pptuser
COPY package*.json ./
COPY . .
EXPOSE 3000

12
app.vue Normal file
View File

@ -0,0 +1,12 @@
<template>
<div>
<NuxtPage />
</div>
</template>
<script setup>
</script>
<style>
/* Estilos globales */
</style>

84
bak/baxk1.js Normal file
View File

@ -0,0 +1,84 @@
// utils/filterProperties.js
import Fuse from 'fuse.js';
export const filterSimilarProperties = (properties) => {
const options = {
keys: ['dataValues.title'], // Buscamos similitud en el título dentro de dataValues
threshold: 0.6, // 0.6 es un umbral para 40% de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(properties, options);
const uniqueProperties = [];
const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
properties.forEach((property) => {
const propertyData = property.dataValues;
if (!seen.has(propertyData.id)) {
// Inicializa el array para IDs similares en dataValues
propertyData.similarIds = [];
// Buscamos propiedades similares por título
const result = fuse.search(propertyData.title).filter((res) => {
const similarProperty = res.item.dataValues;
// Filtramos resultados que no sean la misma propiedad y sean similares en título
const isSimilar = res.item.dataValues.id !== propertyData.id && res.score < options.threshold;
if (isSimilar) {
propertyData.similarIds.push(similarProperty.id);
// Enriquecer la propiedad actual con datos de la propiedad similar
propertyData.title = getLongest(propertyData.title, similarProperty.title);
propertyData.url = propertyData.url || similarProperty.url;
propertyData.price = propertyData.price || similarProperty.price;
propertyData.rooms = propertyData.rooms || similarProperty.rooms;
propertyData.area = propertyData.area || similarProperty.area;
propertyData.level = propertyData.level || similarProperty.level;
propertyData.description = getLongest(propertyData.description, similarProperty.description);
propertyData.pic = propertyData.pic || similarProperty.pic;
propertyData.baths = propertyData.baths || similarProperty.baths;
propertyData.neighborhood = propertyData.neighborhood || similarProperty.neighborhood;
propertyData.phone = propertyData.phone || similarProperty.phone;
// Marcar la propiedad similar como vista
seen.add(similarProperty.id);
}
return isSimilar;
});
// Comprobamos coincidencia exacta en price, rooms y area
const matchingProperties = result.filter((res) => {
const similarProperty = res.item.dataValues;
return (
similarProperty.price === propertyData.price &&
(similarProperty.rooms === propertyData.rooms ||
similarProperty.area === propertyData.area)
);
});
if (matchingProperties.length === 0) {
uniqueProperties.push(property);
}
// Añadir los IDs de propiedades similares al array similarIds y marcarlas como vistas
matchingProperties.forEach((res) => {
const similarProperty = res.item.dataValues;
propertyData.similarIds.push(similarProperty.id);
seen.add(similarProperty.id);
});
// Añadir la propiedad actual como vista
seen.add(propertyData.id);
}
});
return uniqueProperties;
};
const getLongest = (a, b) => {
if (!a) return b;
if (!b) return a;
return a.length >= b.length ? a : b;
};

57
bak/completePropertyData1.js Normal file
View File

@ -0,0 +1,57 @@
// utils/completePropertyData.js
import Fuse from 'fuse.js';
export const completePropertyData = (uniqueProperties, allProperties) => {
// Creamos una copia de allProperties con sólo los dataValues
const allDataValues = allProperties.map(prop => prop.dataValues);
const options = {
keys: ['title'], // Buscamos similitud en el título
threshold: 0.6, // Umbral de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(allDataValues, options);
const enrichedProperties = uniqueProperties.map(property => {
// Buscamos propiedades similares por título para completar datos
const result = fuse.search(property.title).filter((res) => {
// Filtramos resultados que no sean la misma propiedad y sean similares en título
return res.item.id !== property.id && res.score < options.threshold;
});
// Combinamos las propiedades similares para completar datos
let enrichedProperty = { ...property };
result.forEach((res) => {
const item = res.item;
enrichedProperty = {
...enrichedProperty,
title: getLongest(enrichedProperty.title, item.title),
url: enrichedProperty.url || item.url,
price: enrichedProperty.price || item.price,
rooms: enrichedProperty.rooms || item.rooms,
area: enrichedProperty.area || item.area,
level: enrichedProperty.level || item.level,
description: getLongest(enrichedProperty.description, item.description),
pic: enrichedProperty.pic || item.pic,
baths: enrichedProperty.baths || item.baths,
neighborhood: enrichedProperty.neighborhood || item.neighborhood,
phone: enrichedProperty.phone || item.phone,
createdAt: enrichedProperty.createdAt || item.createdAt,
updatedAt: enrichedProperty.updatedAt || item.updatedAt
};
});
return enrichedProperty;
});
return enrichedProperties;
};
// Función auxiliar para obtener el texto más largo o el que no sea nulo
const getLongest = (a, b) => {
if (!a) return b;
if (!b) return a;
return a.length >= b.length ? a : b;
};

62
bak/filterProperties copy.js Normal file
View File

@ -0,0 +1,62 @@
// utils/filterProperties.js
import Fuse from 'fuse.js';
export const filterSimilarProperties = (properties) => {
const options = {
keys: ['title'], // Solo buscamos similitud en el título
threshold: 0.6, // 0.6 es un umbral para 40% de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(properties, options);
const uniqueProperties = [];
const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
properties.forEach((property) => {
const propertyData = property.dataValues;
if (!seen.has(propertyData.id)) {
// Inicializa el array para IDs similares en dataValues
propertyData.similarIds = [];
// Buscamos propiedades similares por título
const result = fuse.search(propertyData.title).filter((res) => {
const similarProperty = res.item.dataValues;
// Filtramos resultados que no sean la misma propiedad y sean similares en título
const isSimilar = res.item.dataValues.id !== propertyData.id && res.score < options.threshold;
if (isSimilar) {
propertyData.similarIds.push(similarProperty.id);
// Enriquecer la propiedad actual con datos de la propiedad similar
// propertyData.title = getLongest(propertyData.title, similarProperty.title);
// propertyData.url = propertyData.url || similarProperty.url;
// propertyData.price = propertyData.price || similarProperty.price;
// propertyData.rooms = propertyData.rooms || similarProperty.rooms;
// propertyData.area = propertyData.area || similarProperty.area;
// propertyData.level = propertyData.level || similarProperty.level;
// propertyData.description = getLongest(propertyData.description, similarProperty.description);
// propertyData.pic = propertyData.pic || similarProperty.pic;
// propertyData.baths = propertyData.baths || similarProperty.baths;
// propertyData.neighborhood = propertyData.neighborhood || similarProperty.neighborhood;
// propertyData.phone = propertyData.phone || similarProperty.phone;
// Marcar la propiedad similar como vista
//seen.add(similarProperty.id);
}
return isSimilar;
});
uniqueProperties.push(property);
}
});
return uniqueProperties;
};
const getLongest = (a, b) => {
if (!a) return b;
if (!b) return a;
return a.length >= b.length ? a : b;
};

48
bak/filterProperties1.js Normal file
View File

@ -0,0 +1,48 @@
// utils/filterProperties.js
import Fuse from 'fuse.js';
export const filterSimilarProperties = (properties) => {
const options = {
keys: ['title'], // Solo buscamos similitud en el título
threshold: 0.6, // 0.6 es un umbral para 40% de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(properties, options);
const uniqueProperties = [];
const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
properties.forEach((property) => {
if (!seen.has(property.id)) {
// Buscamos propiedades similares por título
const result = fuse.search(property.title).filter((res) => {
// Filtramos resultados que no sean la misma propiedad y sean similares en título
return res.item.id !== property.id && res.score < options.threshold;
});
// Comprobamos coincidencia exacta en price, rooms y area
const matchingProperties = result.filter((res) => {
const item = res.item;
return (
item.price === property.price &&
(item.rooms === property.rooms ||
item.area === property.area)
);
});
if (matchingProperties.length === 0) {
uniqueProperties.push(property);
}
// Marcar propiedades similares como vistas
matchingProperties.forEach((res) => {
seen.add(res.item.id);
});
// Añadir la propiedad actual como vista
seen.add(property.id);
}
});
return uniqueProperties;
};

48
bak/filterProperties_bak.js Normal file
View File

@ -0,0 +1,48 @@
// utils/filterProperties.js
import Fuse from 'fuse.js';
export const filterSimilarProperties = (properties) => {
const options = {
keys: ['title'], // Solo buscamos similitud en el título
threshold: 0.6, // 0.6 es un umbral para 40% de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(properties, options);
const uniqueProperties = [];
const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
properties.forEach((property) => {
if (!seen.has(property.id)) {
// Buscamos propiedades similares por título
const result = fuse.search(property.title).filter((res) => {
// Filtramos resultados que no sean la misma propiedad y sean similares en título
return res.item.id !== property.id && res.score < options.threshold;
});
// Comprobamos coincidencia exacta en price, rooms y area
const matchingProperties = result.filter((res) => {
const item = res.item;
return (
item.price === property.price &&
(item.rooms === property.rooms ||
item.area === property.area)
);
});
if (matchingProperties.length === 0) {
uniqueProperties.push(property);
}
// Marcar propiedades similares como vistas
matchingProperties.forEach((res) => {
seen.add(res.item.id);
});
// Añadir la propiedad actual como vista
seen.add(property.id);
}
});
return uniqueProperties;
};

56
bak/fotocasaScraper.js Normal file
View File

@ -0,0 +1,56 @@
import puppeteer from 'puppeteer';
import Listing from '../../server/db/models/Listing';
//import { notifyNewListing } from './telegramBot';
export default async function scrapeFotocasa() {
console.log('Starting Fotocasa scraping process');
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
const page = await browser.newPage();
await page.goto('https://www.fotocasa.es/es/comprar/viviendas/granada-capital/todas-las-zonas/l');
const pageSourceHTML = await page.content();
await browser.close();
console.log('Navigated to Fotocasa',pageSourceHTML);
try {
const listings = await page.evaluate(() => {
const results = [];
document.querySelectorAll('.re-CardPackPremium').forEach((element) => {
const title = element.querySelector('.re-CardTitle')?.innerText.trim();
const priceText = element.querySelector('.re-CardPrice')?.innerText.trim();
const price = parseFloat(priceText.replace(/[^\d]/g, ''));
const location = title;
const description = element.querySelector('.re-CardDescription-text')?.innerText.trim();
const url = element.querySelector('a')?.href;
if (title && price && location && description && url) {
results.push({ title, price, location, description, url });
}
console.log(title, price, location, description, url);
});
return results;
});
console.log(`Found ${listings.length} listings`);
for (const listingData of listings) {
try {
const listing = new Listing(listingData);
//await listing.save();
console.log(`Saved listing to database: ${listing.title}`);
//await notifyNewListing(listing);
console.log(`Sent notification for listing: ${listing.title}`);
} catch (innerError) {
console.error(`Error processing listing: ${innerError.message}`);
}
}
} catch (error) {
console.error('Error scraping Fotocasa:', error.message);
throw new Error('Failed to scrape Fotocasa');
} finally {
await browser.close();
console.log('Finished Fotocasa scraping process');
}
}

76
bak/habitacliaScraper.js Normal file
View File

@ -0,0 +1,76 @@
//import { notifyNewListing } from './telegramBot';
// import Listing from '../../server/db/models/Listing';
export default async function scrapeHabitaclia(puppeteer) {
console.log("Starting Habitaclia scraping process");
const url = 'https://idealista7.p.rapidapi.com/listhomes?order=relevance&operation=sale&locationId=0-EU-ES-28-07-001-079&locationName=Madrid&numPage=1&maxItems=40&location=es&locale=en';
const options = {
method: 'GET',
headers: {
'x-rapidapi-key': '080e9362e4mshdbcf5473f82adb9p196205jsne6a02c4e6381',
'x-rapidapi-host': 'idealista7.p.rapidapi.com'
}
};
try {
const response = await fetch(url, options);
const result = await response.text();
console.log(result);
} catch (error) {
console.error(error);
}
// const browser = await puppeteer.launch({
// headless: true,
// args: [
// '--no-sandbox' ,
// '--disable-setuid-sandbox'
// ]
// });
// const page = await browser.newPage();
// console.log(puppeteer, browser);
try {
// await page.goto('https://www.habitaclia.com/viviendas-granada_ciudad.htm', {
// waitUntil: 'networkidle2',
// });
// console.log('Navigated to Habitaclia');
// const content = await page.content();
// console.log(content);
// await page.screenshot({ path: 'habitaclia_state.png', fullPage: true })
// const listings = await page.evaluate(() => {
// const results = [];
// document.querySelectorAll('.list-item').forEach((element) => {
// const title = element.querySelector('.list-item-title')?.innerText.trim();
// const priceText = element.querySelector('.list-item-price')?.innerText.trim();
// const price = parseFloat(priceText.replace(/[^\d]/g, ''));
// const location = element.querySelector('.list-item-location')?.innerText.trim();
// const description = element.querySelector('.list-item-description')?.innerText.trim();
// const url = element.querySelector('.list-item-title a')?.href;
// if (title && price && location && description && url) {
// results.push({ title, price, location, description, url });
// }
// });
// return results;
// });
// console.log(`Found ${listings.length} listings`);
// for (const listingData of listings) {
// try {
// const listing = new Listing(listingData);
// //await listing.save();
// console.log(`Saved listing to database: ${listing.title}`);
// //await notifyNewListing(listing);
// console.log(`Sent notification for listing: ${listing.title}`);
// } catch (innerError) {
// console.error(`Error processing listing: ${innerError.message}`);
// }
// }
} catch (error) {
console.error("Error scraping Habitaclia:", error.message);
throw new Error("Failed to scrape Habitaclia");
} finally {
await browser.close();
console.log("Finished Habitaclia scraping process");
}
}

43
bak/index.cjs Normal file
View File

@ -0,0 +1,43 @@
'use strict';
const fs = require('fs');
const path = require('path');
const Sequelize = require('sequelize');
const process = require('process');
const basename = path.basename(__filename);
const env = process.env.NODE_ENV || 'development';
const config = require(__dirname + '/../../../../config/sequelize.js')[env];
const db = {};
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(config.database, config.username, config.password, config);
}
fs
.readdirSync(__dirname)
.filter(file => {
return (
file.indexOf('.') !== 0 &&
file !== basename &&
file.slice(-3) === '.js' &&
file.indexOf('.test.js') === -1
);
})
.forEach(file => {
const model = require(path.join(__dirname, file))(sequelize, Sequelize.DataTypes);
db[model.name] = model;
});
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
db[modelName].associate(db);
}
});
db.sequelize = sequelize;
db.Sequelize = Sequelize;
module.exports = db;

5
bak/listings.ts Normal file
View File

@ -0,0 +1,5 @@
// import Listing from '../db/models/Listing';
export default defineEventHandler(async () => {
// return await Listing.find();
});

68
bak/nitroPlugin.ts Normal file
View File

@ -0,0 +1,68 @@
/* import {DEFAULT_INTERCEPT_RESOLUTION_PRIORITY} from 'puppeteer';
import Puppeteer from 'puppeteer';
import { addExtra } from 'puppeteer-extra';
import StealthPlugin from "puppeteer-extra-plugin-stealth";
import AdblockerPlugin from "puppeteer-extra-plugin-adblocker";
import chromeApp from "puppeteer-extra-plugin-stealth/evasions/chrome.app";
import chromeCsi from "puppeteer-extra-plugin-stealth/evasions/chrome.csi";
import chromeLoadTimes from "puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes";
import chromeRuntime from "puppeteer-extra-plugin-stealth/evasions/chrome.runtime";
import defaultArgs from "puppeteer-extra-plugin-stealth/evasions/defaultArgs";
import iframeContentWindow from "puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow";
import mediaCodecs from "puppeteer-extra-plugin-stealth/evasions/media.codecs";
import navigatorHardwareConcurrency from "puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency";
import navigatorLanguages from "puppeteer-extra-plugin-stealth/evasions/navigator.languages";
import navigatorPermissions from "puppeteer-extra-plugin-stealth/evasions/navigator.permissions";
import navigatorPlugins from "puppeteer-extra-plugin-stealth/evasions/navigator.plugins";
import navigatorVendor from "puppeteer-extra-plugin-stealth/evasions/navigator.vendor";
import navigatorWebdriver from "puppeteer-extra-plugin-stealth/evasions/navigator.webdriver";
import sourceUrl from "puppeteer-extra-plugin-stealth/evasions/sourceurl";
import userAgentOverride from "puppeteer-extra-plugin-stealth/evasions/user-agent-override";
import webglVendor from "puppeteer-extra-plugin-stealth/evasions/webgl.vendor";
import windowOuterDimensions from "puppeteer-extra-plugin-stealth/evasions/window.outerdimensions";
export default defineNitroPlugin((nitroApp) => {
console.log("Loading puppeteer plugin...");
const puppeteer = addExtra(Puppeteer);
const stealth = StealthPlugin();
configureEvasions(stealth);
puppeteer.use(stealth);
puppeteer.use(
AdblockerPlugin({
// Optionally enable Cooperative Mode for several request interceptors
interceptResolutionPriority: DEFAULT_INTERCEPT_RESOLUTION_PRIORITY
})
)
nitroApp.hooks.hook("request", (event) => {
event.context.$puppeteer = puppeteer;
});
});
// Configurar evasiones
const configureEvasions = (stealth) => {
// Clear existing evasions
stealth.enabledEvasions.clear();
// Add evasions explicitly
stealth.enabledEvasions.add(chromeApp);
stealth.enabledEvasions.add(chromeCsi);
stealth.enabledEvasions.add(chromeLoadTimes);
stealth.enabledEvasions.add(chromeRuntime);
stealth.enabledEvasions.add(defaultArgs);
stealth.enabledEvasions.add(iframeContentWindow);
stealth.enabledEvasions.add(mediaCodecs);
stealth.enabledEvasions.add(navigatorHardwareConcurrency);
stealth.enabledEvasions.add(navigatorLanguages);
stealth.enabledEvasions.add(navigatorPermissions);
stealth.enabledEvasions.add(navigatorPlugins);
stealth.enabledEvasions.add(navigatorVendor);
stealth.enabledEvasions.add(navigatorWebdriver);
stealth.enabledEvasions.add(sourceUrl);
stealth.enabledEvasions.add(userAgentOverride);
stealth.enabledEvasions.add(webglVendor);
stealth.enabledEvasions.add(windowOuterDimensions);
}; */

55
bak/pisoscomScraper.js Normal file
View File

@ -0,0 +1,55 @@
import puppeteer from 'puppeteer';
import Listing from '../../server/db/models/Listing';
//import { notifyNewListing } from './telegramBot';
export default async function scrapePisoscom() {
console.log('Starting Pisos.com scraping process');
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
const page = await browser.newPage();
try {
await page.goto('https://www.pisos.com/venta/pisos-area_de_granada_granada_capital/', {
waitUntil: 'networkidle2',
});
console.log('Navigated to Pisos.com');
const listings = await page.evaluate(() => {
const results = [];
document.querySelectorAll('.ad-preview').forEach((element) => {
const title = element.querySelector('.ad-preview__title').innerText.trim();
const priceText = element.querySelector('.ad-preview__price').innerText.trim();
const price = parseFloat(priceText.replace(/[^\d]/g, ''));
const location = element.querySelector('.ad-preview__subtitle').innerText.trim();
const description = element.querySelector('.ad-preview__description').innerText.trim();
const phone = element.querySelector('.contact-box__phone').getAttribute('data-number');
const url = element.getAttribute('data-lnk-href');
if (title && price && location && description && url) {
results.push({ title, price, location, description, url, phone });
}
});
return results;
});
console.log(`Found ${listings.length} listings. Listing data: ${JSON.stringify(listings)}`);
for (const listingData of listings) {
try {
const listing = new Listing(listingData);
//await listing.save();
console.log(`Saved listing to database: ${listing.title}`);
//await notifyNewListing(listing);
console.log(`Sent notification for listing: ${listing.title}`);
} catch (innerError) {
console.error(`Error processing listing: ${innerError.message}`);
}
}
} catch (error) {
console.error('Error scraping Pisos.com:', error.message);
throw new Error('Failed to scrape Pisos.com');
} finally {
await browser.close();
console.log('Finished Pisos.com scraping process');
}
}

51
bak/scrape.vue Normal file
View File

@ -0,0 +1,51 @@
<template>
<div class="flex items-center justify-center min-h-screen bg-gray-100">
<div class="bg-white p-6 rounded-lg shadow-lg text-center max-w-md w-full">
<h1 class="text-2xl font-bold mb-4">Initiate Scraping</h1>
<button
@click="scrape"
class="bg-blue-500 text-white font-semibold py-2 px-4 rounded hover:bg-blue-700 transition-colors duration-300"
>
Start Scraping
</button>
<p v-if="message" class="mt-4 text-green-600">{{ message }}</p>
</div>
</div>
</template>
<script setup>
import { ref } from 'vue';
const message = ref('');
const scrapers= [
//'scrapepisoscom',
'scrapehabitaclia'
]
const scrape = async () => {
message.value = 'Starting scraping...\n';
for (const portal of scrapers) {
await initiateScraping(portal);
}
};
const initiateScraping = async (portal) => {
try {
const response = await fetch(`/api/${portal}`, {
method: 'POST',
});
const result = await response.json();
if (result.success) {
message.value = 'Scraping initiated successfully: ' + JSON.stringify(result);
} else {
message.value = `Error: ${result.message}`;
}
} catch (error) {
message.value = `Error: ${error.message}`;
}
};
</script>
<style>
/* Tailwind CSS se encargará de los estilos, por lo que no se necesita nada aquí */
</style>

11
bak/scrapehabitaclia.js Normal file
View File

@ -0,0 +1,11 @@
import scrapeHabitaclia from '../services/scraper/habitacliaScraper';
export default defineEventHandler(async (event) => {
const puppeteer = event.context.$puppeteer;
try {
const result = await scrapeHabitaclia(puppeteer);
return { success: true, message: 'Habitacia scraping initiated.' , data: result || {}};
} catch (error) {
return { success: false, message: 'Failed to initiate Habitacia scraping.', error: error.message };
}
});

10
bak/scrapepisoscom.js Normal file
View File

@ -0,0 +1,10 @@
// import pisoscomScraper from '../../modules/scraping/pisoscomScraper';
export default defineEventHandler(async (event) => {
try {
const result = await pisoscomScraper();
return { success: true, message: 'Fotocasa scraping initiated.' , data: result || {}};
} catch (error) {
return { success: false, message: 'Failed to initiate Fotocasa scraping.', error: error.message };
}
});

15
bak/test.ts Normal file
View File

@ -0,0 +1,15 @@
import { createClient } from 'pexels';
const client = createClient('dKrFwkztM2MT9ss60BIBukRyTQpwEN4cFdydJis2zi2xeQ1NC4irqWGl');
export default defineEventHandler(async (event) => {
const {query} = await readBody(event);
const result = await client.photos.search({ query, per_page: 20 });
return result?.photos.map(item => {
return {
id :item.id,
alt: item.alt,
src: item.src.large
};
});
});

9
bak/useDatabase.js Normal file
View File

@ -0,0 +1,9 @@
import mongoose from 'mongoose';
export const useDatabase = () => {
const config = useRuntimeConfig();
mongoose.connect(config.mongodbUri, { useNewUrlParser: true, useUnifiedTopology: true });
return mongoose;
}

115
components/Card.vue Normal file
View File

@ -0,0 +1,115 @@
<template>
<div class="card"
:class="{ dragging: isDragging }"
ref="card"
:style="{ transform: `translate(${offsetX}px, ${offsetY}px)`, opacity: cardOpacity }"
@mousedown="handleMouseDown"
@touchstart="handleTouchStart"
@touchmove="handleTouchMove"
@touchend="handleTouchEnd">
<div class="content">
<h2 class="text-xl font-bold mb-2">{{ property.title }}</h2>
<img :src="property.pic" alt="Property Image" class="mb-4 w-full h-48 object-cover rounded-lg" />
<p class="text-gray-700 mb-2">{{ property.description }}</p>
<p class="text-gray-900 font-bold">Price: {{ property.price }}</p>
<p class="text-gray-700">Rooms: {{ property.rooms }} | Baths: {{ property.baths }} | Area: {{ property.area }} sqft</p>
<p class="text-gray-600 text-sm mt-2">Location: {{ property.neighborhood }}</p>
</div>
</div>
</template>
<script>
export default {
props: {
property: {
type: Object,
required: true
}
},
data() {
return {
startX: 0,
startY: 0,
offsetX: 0,
offsetY: 0,
isDragging: false,
cardOpacity: 1
};
},
methods: {
handleMouseDown(event) {
this.isDragging = true;
this.startX = event.clientX;
this.startY = event.clientY;
document.addEventListener('mousemove', this.handleMouseMove);
document.addEventListener('mouseup', this.handleMouseUp);
},
handleMouseMove(event) {
if (this.isDragging) {
const moveX = event.clientX - this.startX;
const moveY = event.clientY - this.startY;
this.offsetX = moveX;
this.offsetY = moveY;
this.updateOpacity();
}
},
handleMouseUp() {
this.isDragging = false;
document.removeEventListener('mousemove', this.handleMouseMove);
document.removeEventListener('mouseup', this.handleMouseUp);
if (Math.abs(this.offsetX) > window.innerWidth / 3) {
this.cardOpacity = 0;
} else {
this.offsetX = 0;
this.offsetY = 0;
this.cardOpacity = 1;
}
},
handleTouchStart(event) {
this.startX = event.touches[0].clientX;
this.startY = event.touches[0].clientY;
},
handleTouchMove(event) {
const moveX = event.touches[0].clientX - this.startX;
const moveY = event.touches[0].clientY - this.startY;
this.offsetX = moveX;
this.offsetY = moveY;
this.updateOpacity();
},
handleTouchEnd() {
if (Math.abs(this.offsetX) > window.innerWidth / 2.5) {
this.cardOpacity = 0;
} else {
this.offsetX = 0;
this.offsetY = 0;
this.cardOpacity = 1;
}
},
updateOpacity() {
const windowWidth = window.innerWidth;
const cardWidth = this.$refs.card.clientWidth;
const distanceToEdge = Math.min(Math.abs(this.offsetX), windowWidth - cardWidth);
this.cardOpacity = 1 - (distanceToEdge / (windowWidth / 2));
}
}
};
</script>
<style scoped>
.card {
color: black;
position: relative;
width: 30%;
height: 80%;
background-color: #ffffff;
border-radius: 10px;
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.2);
transition: transform 0.3s cubic-bezier(0.25, 0.1, 0.25, 1), opacity 0.3s;
user-select: none;
cursor: pointer;
}
.content {
padding: 20px;
}
</style>

29
config/sequelize.js Normal file
View File

@ -0,0 +1,29 @@
import dotenv from 'dotenv';
dotenv.config();
const sequelizeConfig = {
development: {
username: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD ,
database: process.env.MYSQL_DATABASE,
host: "thax.es",
dialect: 'mysql',
},
test: {
username: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD ,
database: process.env.MYSQL_DATABASE,
host: process.env.MYSQL_HOST,
dialect: 'mysql',
},
production: {
username: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD ,
database: process.env.MYSQL_DATABASE,
host: process.env.MYSQL_HOST,
dialect: 'mysql',
},
};
export default sequelizeConfig;

6
eslint.config.mjs Normal file
View File

@ -0,0 +1,6 @@
// @ts-check
import withNuxt from './.nuxt/eslint.config.mjs'
export default withNuxt(
// Your custom configs here
)

40
nuxt.config.ts Normal file
View File

@ -0,0 +1,40 @@
//import scrapingModule from './modules/scraping';
// https://nuxt.com/docs/api/configuration/nuxt-config
import path from 'path';
export default defineNuxtConfig({
alias: {
},
colorMode: {
preference: 'light'
},
devtools: { enabled: true },
modules: [
"@nuxt/ui",
"@nuxtjs/color-mode",
"@nuxtjs/tailwindcss",
"@nuxt/eslint",
],
components: {
global: true,
dirs: ["~/components"],
},
runtimeConfig: {
mongodbUri: process.env.MONGODB_URI,
telegramBotToken: process.env.TELEGRAM_BOT_TOKEN,
telegramChatId: process.env.TELEGRAM_CHAT_ID,
mysqlHost: process.env.MYSQL_HOST,
mysqlUser: process.env.MYSQL_USER,
mysqlPassword: process.env.MYSQL_PASSWORD,
mysqlDatabase: process.env.MYSQL_DATABASE,
},
postcss: {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
},
nitro: {
//plugins: ['/server/plugins/db/init.ts']
},
});

18864
package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

55
package.json Normal file
View File

@ -0,0 +1,55 @@
{
"name": "nuxt-app",
"private": true,
"type": "module",
"scripts": {
"build": "nuxt build",
"dev": "npm install && nuxt dev",
"generate": "nuxt generate",
"preview": "nuxt preview",
"postinstall": "nuxt prepare",
"lint": "eslint .",
"start": "node .output/server/index.mjs"
},
"dependencies": {
"@nuxt/eslint": "^0.3.12",
"@nuxt/kit": "^3.11.2",
"@nuxt/ui": "^2.14.2",
"@rollup/plugin-commonjs": "^25.0.8",
"@rollup/plugin-node-resolve": "^15.2.3",
"axios": "^0.21.1",
"cheerio": "^1.0.0-rc.10",
"d2l-intl": "^2.1.0",
"fuse.js": "^7.0.0",
"mongoose": "^5.12.3",
"mysql2": "^3.10.0",
"node-cron": "^3.0.0",
"node-telegram-bot-api": "^0.66.0",
"nuxt": "^3.11.1",
"pexels": "^1.4.0",
"puppeteer": "^22.10.0",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-adblocker": "^2.13.6",
"puppeteer-extra-plugin-anonymize-ua": "^2.4.6",
"puppeteer-extra-plugin-block-resources": "^2.4.3",
"puppeteer-extra-plugin-recaptcha": "^3.6.8",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"sequelize": "^6.37.3",
"sequelize-mig": "^3.1.3",
"string-similarity": "^4.0.4",
"vue": "^3.4.21",
"vue-router": "^4.3.0"
},
"devDependencies": {
"@nuxtjs/tailwindcss": "^6.11.4",
"@types/cheerio": "^0.22.35",
"eslint": "^8.57.0",
"eslint-config-prettier": "^9.1.0",
"prettier": "^3.2.5",
"prettier-plugin-tailwindcss": "^0.5.12",
"sequelize-auto-migrations": "^1.0.3",
"sequelize-cli": "^6.6.2",
"typescript": "^5.4.5",
"vue-tsc": "^1.8.27"
}
}

25
pages/index.vue Normal file
View File

@ -0,0 +1,25 @@
<template>
<div>
<h1>Real Estate Listings</h1>
<ul>
<li v-for="listing in listings" :key="listing._id">
<h2>{{ listing.title }}</h2>
<p>Price: {{ listing.price }}</p>
<p>Location: {{ listing.location }}</p>
<p>Description: {{ listing.description }}</p>
<a :href="listing.url">View Listing</a>
</li>
</ul>
</div>
</template>
<script setup>
import { ref, onMounted } from 'vue';
const listings = ref([]);
onMounted(async () => {
// const response = await $fetch('/api/listings');
// listings.value = response;
});
</script>

89
pages/list.vue Normal file
View File

@ -0,0 +1,89 @@
<template>
<div class="container mx-auto py-8">
<h1 class="text-3xl font-bold mb-8 text-center">Property Listings</h1>
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-6">
<Card
v-for="property in properties"
:key="property.id"
:property="property"
class="card"
>
</Card>
</div>
</div>
</template>
<script setup>
import { ref, onMounted } from "vue";
const properties = ref([]);
const fetchProperties = async () => {
try {
const response = await fetch("/api/house/construe", {
method: "GET",
});
const result = await response.json();
if (result.success) {
properties.value = result.data;
} else {
console.error(result.message);
}
} catch (error) {
console.error("Error fetching properties:", error);
}
};
onMounted(() => {
fetchProperties();
});
</script>
<style scoped>
h1 {
font-size: x-large;
font-weight: 700;
text-align: center;
}
.card {
position: absolute;
top: 0;
left: 0;
right: 0;
bottom: 0;
margin: auto;
display: flex;
flex-direction: column;
justify-content: top;
align-items: center;
overflow: hidden;
}
.blur {
opacity: 0.6;
filter: blur(10px);
backdrop-filter: blur(
10px
); /* Propiedad experimental para algunos navegadores */
}
.card .content {
padding: 20px;
}
img {
pointer-events: none;
-webkit-user-drag: none;
-khtml-user-drag: none;
-moz-user-drag: none;
-o-user-drag: none;
user-drag: none;
}
.card img {
width: 100%;
height: 100%;
object-fit: cover;
}
</style>

50
pages/parse.vue Normal file
View File

@ -0,0 +1,50 @@
<template>
<div class="flex items-center justify-center min-h-screen bg-gray-100">
<div class="bg-white p-6 rounded-lg shadow-lg text-center max-w-md w-full">
<h1 class="text-2xl font-bold mb-4">Initiate Parse</h1>
<button
@click="parse"
class="bg-blue-500 text-white font-semibold py-2 px-4 rounded hover:bg-blue-700 transition-colors duration-300"
>
Start Parse
</button>
<p v-if="message" class="mt-4 text-green-600">{{ message }}</p>
</div>
</div>
</template>
<script setup>
import { ref } from "vue";
const message = ref("");
const parse = async () => {
message.value = "Starting parse...\n";
const resp = await initiateParse();
};
const initiateParse = async () => {
try {
const response = await fetch("/api/parse", {
method: "POST",
body: JSON.stringify({ id: "searchtext", title: "Test" }),
headers: {
"Content-Type": "application/json",
},
});
const result = await response.json();
if (result.success) {
message.value = 'Parse initiated successfully: ' + JSON.stringify(result);
} else {
message.value = `Error: ${result.message}`;
}
} catch (error) {
message.value = `Error: ${error.message}`;
}
};
</script>
<style>
/* Tailwind CSS se encargará de los estilos, por lo que no se necesita nada aquí */
</style>

BIN
project.zip Normal file
View File

Binary file not shown.

BIN
public/favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.2 KiB

18
server/api/house/construe.get.ts Normal file
View File

@ -0,0 +1,18 @@
import { House } from "../../db/mysql/db.config";
import { ConstrueHouseCommand } from "../../application/command/house/construeHouseCommand";
import { ConstrueHouseCommandHandler } from "../../application/commandHandler/house/construeHouseCommandHandler";
export default defineEventHandler(async (event) => {
try {
const candidateHouses = await House.findAll();
const construeHouseCommandHandler = await ConstrueHouseCommandHandler.create(
ConstrueHouseCommand.create(candidateHouses)
);
return { success: true, data: construeHouseCommandHandler };
} catch (error) {
console.error("Error listing houses:", error);
return { success: false, message: error.message };
}
});

13
server/api/house/list.get.ts Normal file
View File

@ -0,0 +1,13 @@
import { Property } from "../../db/mysql/db.config";
import { groupSimilarProperties } from "../../services/fusejs/filterProperties";
export default defineEventHandler(async (event) => {
try {
const candidateProperties = await Property.findAll();
const properties = groupSimilarProperties(candidateProperties);
return { success: true, properties };
} catch (error) {
console.error("Error listing properties:", error);
return { success: false, message: error.message };
}
});

12
server/api/house/parse.get.ts Normal file
View File

@ -0,0 +1,12 @@
import { Property } from "../../db/mysql/db.config";
export default defineEventHandler(async (event) => {
try {
const body = await readBody(event);
const newProperty = await Property.create(body);
return { success: true, property: newProperty };
} catch (error) {
console.error("Error creating property:", error);
return { success: false, message: error.message };
}
});

100
server/application/command/house/HouseCommand.ts Normal file
View File

@ -0,0 +1,100 @@
class CreateHouseCommand {
private _id : string;
private _title : string;
private _url : string;
private _neighborhood: string;
private _area : string;
private _rooms : string;
private _price : string;
private _description : string;
private _pic : string;
private _baths : string;
private _level : string;
private _phone : string;
private constructor() {
return this;
}
public static create(
id : string,
title : string,
url : string,
neighborhood: string,
area : string,
rooms : string,
price : string,
description : string,
pic : string,
baths : string,
level : string,
phone : string
) {
const createHouseCommand = new CreateHouseCommand();
createHouseCommand._id = id;
createHouseCommand._title = title;
createHouseCommand._url = url;
createHouseCommand._neighborhood = neighborhood;
createHouseCommand._area = area;
createHouseCommand._rooms = rooms;
createHouseCommand._price = price;
createHouseCommand._description = description;
createHouseCommand._pic = pic;
createHouseCommand._baths = baths;
createHouseCommand._level = level;
createHouseCommand._phone = phone;
return createHouseCommand;
}
public id() {
return this._id;
}
public title() {
return this._title;
}
public url() {
return this._url;
}
public neighborhood() {
return this._neighborhood;
}
public area() {
return this._area;
}
public rooms() {
return this._rooms;
}
public price() {
return this._price;
}
public description() {
return this._description;
}
public pic() {
return this._pic;
}
public baths() {
return this._baths;
}
public level() {
return this._level;
}
public phone() {
return this._phone;
}
}

22
server/application/command/house/construeHouseCommand.ts Normal file
View File

@ -0,0 +1,22 @@
export class ConstrueHouseCommand {
private _rawHouseList: object;
private constructor() {
return this;
}
public static create(
rawHouseList: object,
) {
const construeHouseCommand = new ConstrueHouseCommand();
construeHouseCommand._rawHouseList = rawHouseList;
return construeHouseCommand;
}
public rawHouseList() {
return this._rawHouseList;
}
}

173
server/application/commandHandler/house/construeHouseCommandHandler.ts Normal file
View File

@ -0,0 +1,173 @@
import { Op, sequelize, House as Hou } from "../../../db/mysql/db.config";
import { House } from "../../../domain/house";
import type { ConstrueHouseCommand } from "../../command/house/construeHouseCommand";
export class ConstrueHouseCommandHandler {
private constructor() {}
public static async create(construeHouseCommand: ConstrueHouseCommand) {
const houseCandidates: House[] = [];
let i = 0;
construeHouseCommand.rawHouseList().forEach((houseCandidate: object) => {
houseCandidates.push(
House.create(
`${++i}`,
houseCandidate.title,
houseCandidate.url,
houseCandidate.neighborhood,
houseCandidate.area,
houseCandidate.rooms,
houseCandidate.price,
houseCandidate.description,
houseCandidate.pic,
houseCandidate.baths,
houseCandidate.level,
houseCandidate.phone
));
});
const proposedHouses = ConstrueHouseCommandHandler.findHouseByTitleAndExactPriceAndRoomsAndBaths(houseCandidates);
return proposedHouses;
}
async execute(): Promise<void> {
}
private static async findHouseByTitleAndExactPriceAndRoomsAndBaths(houseCandidates: any): Promise<String[]> {
const mergedHouses: any[] = [];
// Usamos un bucle for normal para manejar bien el await dentro del ciclo
for (let i = 0; i < houseCandidates.length; i++) {
const houseCandidate = houseCandidates[i];
const sanitizedTitle = houseCandidate.title.replace(/'/g, "\\'");
// Esperamos a que la búsqueda en la base de datos se resuelva
const matchingHouses = await Hou.findAll({
where: {
[Op.and]: [
{ price: houseCandidate.rawPrice },
{ rooms: houseCandidate.rooms },
{ area: houseCandidate.area },
{ baths: houseCandidate.baths },
sequelize.literal(`MATCH(title) AGAINST('${sanitizedTitle}' IN NATURAL LANGUAGE MODE)`)
]
}
});
const resultHouse = ConstrueHouseCommandHandler.mergeHouseProperties(matchingHouses);
// Si hay coincidencias, añadimos los resultados al array de casas encontradas
if (matchingHouses.length > 0) {
mergedHouses.push(resultHouse);
// Eliminamos todas las casas que coincidan con los resultados obtenidos
houseCandidates = houseCandidates.filter((candidate: any) => {
return !matchingHouses.some((matchedHouse: any) => {
// Aquí puedes definir los criterios para eliminar la casa original.
// Por ejemplo, comparando el ID, precio, habitaciones, etc.
return (
candidate.rawPrice === matchedHouse.price &&
candidate.rooms === matchedHouse.rooms &&
candidate.area === matchedHouse.area &&
candidate.baths === matchedHouse.baths
);
});
});
// Como hemos modificado el array, restamos 1 al índice
i = -1; // Reiniciamos el índice para volver a iterar el array actualizado
}
}
return mergedHouses;
// const sanitizedTitle = house.title.replace(/'/g, "\\'");
// const matchingHouses = Hou.findAll({
// where: {
// [Op.and]: [
// {price: house.rawPrice},
// {rooms: house.rooms},
// {area: house.area},
// {baths: house.baths},
// sequelize.literal(`MATCH(title) AGAINST('${sanitizedTitle}' IN NATURAL LANGUAGE MODE)`)
// ]
// }
// });
// return matchingHouses;
}
private static mergeHouseProperties(houses: any[]): any {
if (!Array.isArray(houses)) {
throw new Error("El parámetro 'houses' debe ser un array.");
}
const mergedHouse = {
id: ConstrueHouseCommandHandler.getLongestString(houses.map(h => h.id)) || Math.floor(Math.random() * 100000000).toString().padStart(8, '0'),
title: ConstrueHouseCommandHandler.getLongestString(houses.map(h => h.title)),
url: ConstrueHouseCommandHandler.joinByComma(houses.map(h => h.url)),
neighborhood: ConstrueHouseCommandHandler.getLongestString(houses.map(h => h.neighborhood)),
area: houses[0]?.area || 0, // Asumo que el área es la misma en todos los duplicados
rooms: houses[0]?.rooms || 0, // Asumo que el número de habitaciones es el mismo
price: houses[0]?.price || 0, // Asumo que el precio es el mismo
description: ConstrueHouseCommandHandler.getLongestString(houses.map(h => h.description)),
pic: ConstrueHouseCommandHandler.joinByComma(houses.map(h => h.pic)),
baths: houses[0]?.baths || 0, // Asumo que el número de baños es el mismo
level: ConstrueHouseCommandHandler.getLongestString(houses.map(h => h.level)),
phone: ConstrueHouseCommandHandler.joinByComma(houses.map(h => h.phone)) // Concatenar los teléfonos
};
return mergedHouse;
}
private static getLongestString(strings: string[]): string {
const preSelectedString = strings
.filter(str => str && !str.startsWith('<a href=')) // Ignora las cadenas que comienzan con '<a href='
.reduce((longest, current) => current && current.length > longest.length ? current : longest, "");
const alternativeString = strings.reduce((longest, current) => current && current.length > longest.length ? current : longest, "");
return preSelectedString !== "" ? preSelectedString : ConstrueHouseCommandHandler.parseLinks(alternativeString);
}
private static joinByComma(items: string[]): string {
return items.filter(Boolean).join(', ');
}
private static parseLinks(htmlString) {
// Regex para identificar las etiquetas <a> con el href y el contenido interno
const anchorTagRegex = /<a href="\/comprar\/(.*?)18.*?"[^>]*>(.*?)<\/a>/g;
let match;
let result = "";
while ((match = anchorTagRegex.exec(htmlString)) !== null) {
let urlPart = match[1]; // La parte que queremos desde el href
let innerText = match[2].trim(); // El texto dentro de las etiquetas <a>
// Si innerText está vacío, tomamos la parte de la URL
if (!innerText) {
result= ConstrueHouseCommandHandler.snakeToSpaceCase(urlPart);
} else {
result = innerText;
}
}
return result;
}
private static snakeToSpaceCase(snakeCaseString) {
// Reemplazar tanto los guiones bajos (_) como los guiones medios (-) por espacios
let spaceCaseString = snakeCaseString.replace(/[_-]/g, ' ');
// Capitalizar la primera letra de cada palabra
spaceCaseString = spaceCaseString.split(' ').map(word => {
return word.charAt(0).toUpperCase() + word.slice(1);
}).join(' ');
return spaceCaseString;
}
}

9
server/db/mongo/index.js Normal file
View File

@ -0,0 +1,9 @@
import mongoose from 'mongoose';
export default async () => {
const config = useRuntimeConfig();
await mongoose.connect(config.mongodbUri, { useNewUrlParser: true, useUnifiedTopology: true });
return mongoose;
}

20
server/db/mongo/models/Property.js Normal file
View File

@ -0,0 +1,20 @@
import mongoose from "mongoose";
const propertySchema = new mongoose.Schema({
id: { type: String, required: true, unique: true },
title: { type: String, required: true },
url: { type: String, required: true },
price: { type: String, required: true },
rooms: { type: Number, required: true },
area: { type: Number, required: true },
level: { type: String, required: true },
description: { type: String },
pic: { type: String },
baths: { type: Number, required: true },
neighborhood: { type: String, required: true },
phone: { type: String, required: true },
});
const Property = mongoose.model("Property", propertySchema);
export default Property;

27
server/db/mysql/db.config.js Normal file
View File

@ -0,0 +1,27 @@
import { Sequelize, Op } from 'sequelize';
import sequelizeConfig from '~/config/sequelize';
import defineProperty from "./models/Property";
import defineHouse from "./models/House";
import defineHome from "./models/Home";
const environmentConfig = sequelizeConfig["production"];
const sequelize = new Sequelize(
environmentConfig.database,
environmentConfig.username,
environmentConfig.password,
{
host: environmentConfig.host,
dialect: environmentConfig.dialect,
logging: false,
});
const Property = defineProperty(sequelize);
const House = defineHouse(sequelize);
const Home = defineHome(sequelize);
sequelize.sync()
.then(() => console.log('Database & tables created!'))
.catch((error) => console.error('Error syncing database:', error));
export { sequelize, Op, House, Home, Property };

13
server/db/mysql/index.js Normal file
View File

@ -0,0 +1,13 @@
import {sequelize} from "./db.config";
async function connectToDatabase() {
try {
await sequelize.authenticate();
console.log('Connection to MySQL has been established successfully.');
return sequelize;
} catch (error) {
console.error('Unable to connect to the MySQL database:', error);
}
}
export default connectToDatabase;

103
server/db/mysql/migrations/20240614190129_noname.cjs Normal file
View File

@ -0,0 +1,103 @@
const Sequelize = require("sequelize");
/**
* Actions summary:
*
* createTable() => "properties", deps: []
*
*/
const info = {
revision: 1,
name: "noname",
created: "2024-06-14T19:01:29.478Z",
comment: "",
};
const migrationCommands = (transaction) => [
{
fn: "createTable",
params: [
"properties",
{
id: {
type: Sequelize.STRING,
field: "id",
primaryKey: true,
unique: true,
allowNull: true,
},
title: { type: Sequelize.STRING, field: "title", allowNull: true },
url: { type: Sequelize.STRING, field: "url", allowNull: true },
price: { type: Sequelize.STRING, field: "price", allowNull: true },
rooms: { type: Sequelize.INTEGER, field: "rooms", allowNull: true },
area: { type: Sequelize.INTEGER, field: "area", allowNull: true },
level: { type: Sequelize.STRING, field: "level", allowNull: true },
description: {
type: Sequelize.TEXT,
field: "description",
allowNull: true,
},
pic: { type: Sequelize.TEXT, field: "pic", allowNull: true },
baths: { type: Sequelize.INTEGER, field: "baths", allowNull: true },
neighborhood: {
type: Sequelize.STRING,
field: "neighborhood",
allowNull: true,
},
phone: { type: Sequelize.STRING, field: "phone", allowNull: true },
createdAt: {
type: Sequelize.DATE,
field: "createdAt",
allowNull: false,
},
updatedAt: {
type: Sequelize.DATE,
field: "updatedAt",
allowNull: false,
},
},
{ transaction },
],
},
];
const rollbackCommands = (transaction) => [
{
fn: "dropTable",
params: ["properties", { transaction }],
},
];
const pos = 0;
const useTransaction = true;
const execute = (queryInterface, sequelize, _commands) => {
let index = pos;
const run = (transaction) => {
const commands = _commands(transaction);
return new Promise((resolve, reject) => {
const next = () => {
if (index < commands.length) {
const command = commands[index];
console.log(`[#${index}] execute: ${command.fn}`);
index++;
queryInterface[command.fn](...command.params).then(next, reject);
} else resolve();
};
next();
});
};
if (useTransaction) return queryInterface.sequelize.transaction(run);
return run(null);
};
module.exports = {
pos,
useTransaction,
up: (queryInterface, sequelize) =>
execute(queryInterface, sequelize, migrationCommands),
down: (queryInterface, sequelize) =>
execute(queryInterface, sequelize, rollbackCommands),
info,
};

86
server/db/mysql/migrations/_current.json Normal file
View File

@ -0,0 +1,86 @@
{
"tables": {
"properties": {
"tableName": "properties",
"schema": {
"id": {
"allowNull": true,
"unique": true,
"primaryKey": true,
"field": "id",
"seqType": "Sequelize.STRING"
},
"title": {
"allowNull": true,
"field": "title",
"seqType": "Sequelize.STRING"
},
"url": {
"allowNull": true,
"field": "url",
"seqType": "Sequelize.STRING"
},
"price": {
"allowNull": true,
"field": "price",
"seqType": "Sequelize.STRING"
},
"rooms": {
"allowNull": true,
"field": "rooms",
"seqType": "Sequelize.INTEGER"
},
"area": {
"allowNull": true,
"field": "area",
"seqType": "Sequelize.INTEGER"
},
"level": {
"allowNull": true,
"field": "level",
"seqType": "Sequelize.STRING"
},
"description": {
"allowNull": true,
"field": "description",
"seqType": "Sequelize.TEXT"
},
"pic": {
"allowNull": true,
"field": "pic",
"seqType": "Sequelize.TEXT"
},
"baths": {
"allowNull": true,
"field": "baths",
"seqType": "Sequelize.INTEGER"
},
"neighborhood": {
"allowNull": true,
"field": "neighborhood",
"seqType": "Sequelize.STRING"
},
"phone": {
"allowNull": true,
"field": "phone",
"seqType": "Sequelize.STRING"
},
"createdAt": {
"allowNull": false,
"field": "createdAt",
"seqType": "Sequelize.DATE"
},
"updatedAt": {
"allowNull": false,
"field": "updatedAt",
"seqType": "Sequelize.DATE"
}
},
"indexes": []
}
},
"path": "Z:\\webs\\hometify\\server\\db\\mysql\\migrations\\_current.json",
"backupPath": "Z:\\webs\\hometify\\server\\db\\mysql\\migrations\\_current_bak.json",
"exists": false,
"revision": 1
}

65
server/db/mysql/models/Home.js Normal file
View File

@ -0,0 +1,65 @@
// server/models/Home.js
import { DataTypes } from "sequelize";
export default function House (sequelize) {
return sequelize.define(
"Home",
{
id: {
type: DataTypes.STRING,
allowNull: true,
unique: true,
primaryKey: true,
},
title: {
type: DataTypes.STRING,
allowNull: true,
},
url: {
type: DataTypes.STRING,
allowNull: true,
},
neighborhood: {
type: DataTypes.STRING,
allowNull: true,
},
area: {
type: DataTypes.INTEGER,
allowNull: true,
},
rooms: {
type: DataTypes.INTEGER,
allowNull: true,
},
price: {
type: DataTypes.STRING,
allowNull: true,
},
description: {
type: DataTypes.TEXT,
allowNull: true,
},
pic: {
type: DataTypes.TEXT,
allowNull: true,
},
baths: {
type: DataTypes.INTEGER,
allowNull: true,
},
level: {
type: DataTypes.STRING,
allowNull: true,
},
phone: {
type: DataTypes.STRING,
allowNull: true,
},
},
{
tableName: "home",
timestamps: false, // createdAt y updatedAt
}
);
};

64
server/db/mysql/models/House.js Normal file
View File

@ -0,0 +1,64 @@
// server/models/Property.js
import { DataTypes } from "sequelize";
export default function House (sequelize) {
return sequelize.define(
"House",
{
id: {
type: DataTypes.STRING,
allowNull: true,
unique: true,
primaryKey: true,
},
title: {
type: DataTypes.STRING,
allowNull: true,
},
url: {
type: DataTypes.STRING,
allowNull: true,
},
neighborhood: {
type: DataTypes.STRING,
allowNull: true,
},
area: {
type: DataTypes.INTEGER,
allowNull: true,
},
rooms: {
type: DataTypes.INTEGER,
allowNull: true,
},
price: {
type: DataTypes.STRING,
allowNull: true,
},
description: {
type: DataTypes.TEXT,
allowNull: true,
},
pic: {
type: DataTypes.TEXT,
allowNull: true,
},
baths: {
type: DataTypes.INTEGER,
allowNull: true,
},
level: {
type: DataTypes.STRING,
allowNull: true,
},
phone: {
type: DataTypes.STRING,
allowNull: true,
},
},
{
tableName: "house",
timestamps: false, // createdAt y updatedAt
}
);
};

64
server/db/mysql/models/Property.js Normal file
View File

@ -0,0 +1,64 @@
// server/models/Property.js
import { DataTypes } from "sequelize";
export default function Property (sequelize) {
return sequelize.define(
"Property",
{
id: {
type: DataTypes.STRING,
allowNull: true,
unique: true,
primaryKey: true,
},
title: {
type: DataTypes.STRING,
allowNull: true,
},
url: {
type: DataTypes.STRING,
allowNull: true,
},
price: {
type: DataTypes.STRING,
allowNull: true,
},
rooms: {
type: DataTypes.INTEGER,
allowNull: true,
},
area: {
type: DataTypes.INTEGER,
allowNull: true,
},
level: {
type: DataTypes.STRING,
allowNull: true,
},
description: {
type: DataTypes.TEXT,
allowNull: true,
},
pic: {
type: DataTypes.TEXT,
allowNull: true,
},
baths: {
type: DataTypes.INTEGER,
allowNull: true,
},
neighborhood: {
type: DataTypes.STRING,
allowNull: true,
},
phone: {
type: DataTypes.STRING,
allowNull: true,
},
},
{
tableName: "properties",
timestamps: true, // Incluye createdAt y updatedAt
}
);
};

63
server/db/mysql/models/index.js Normal file
View File

@ -0,0 +1,63 @@
'use strict';
// Importar módulos necesarios
import fs from 'fs';
import path from 'path';
import { fileURLToPath, pathToFileURL } from 'url';
import { Sequelize, DataTypes } from 'sequelize';
import process from 'process';
import configFile from '../../../../config/sequelize.js';
// Definir __filename y __dirname en ES Modules
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
// Variables necesarias
const basename = path.basename(__filename);
const env = process.env.NODE_ENV || 'development';
const config = configFile[env];
const db = {};
// Configuración de Sequelize
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(config.database, config.username, config.password, config);
}
// Función para cargar modelos de forma asíncrona
const loadModels = async () => {
const files = fs.readdirSync(__dirname).filter(file => {
return (
file.indexOf('.') !== 0 &&
file !== basename &&
file.slice(-3) === '.js' &&
file.indexOf('.test.js') === -1
);
});
for (const file of files) {
const modelPath = path.join(__dirname, file);
const modelURL = pathToFileURL(modelPath).href; // Convertir a una URL válida con esquema file://
const { default: modelDef } = await import(modelURL);
const model = modelDef(sequelize, DataTypes);
db[model.name] = model;
}
};
// Ejecutar la carga de modelos
await loadModels();
// Asociar modelos
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
db[modelName].associate(db);
}
});
// Exportar la instancia de Sequelize y modelos
db.sequelize = sequelize;
db.Sequelize = Sequelize;
export default db;

6
server/db/mysql/repository/HomeRepository.ts Normal file
View File

@ -0,0 +1,6 @@
import { sequelize, Op, Home } from "../db.config";
async function insertOrUpdateHome(home: any) {
await Home.create(home);
console.log('Nueva casa insertada:', home.id);
}

18
server/db/mysql/repository/HouseRepository.ts Normal file
View File

@ -0,0 +1,18 @@
import { sequelize, Op, House } from "../db.config";
export async function findHouseByTitleAndExactPriceAndRoomsAndBaths(house: any) {
//const sanitizedTitle = house.title.replace(/'/g, "\\'");
return "pollas";
const matchingHouses = await House.findAll({
where: {
price: house.price,
rooms: house.rooms,
baths: house.baths,
[Op.and]: sequelize.literal(`MATCH(title) AGAINST('${sanitizedTitle}' IN NATURAL LANGUAGE MODE)`)
}
});
return matchingHouses;
}

107
server/domain/house.ts Normal file
View File

@ -0,0 +1,107 @@
export class House {
_id: string;
_title: string | null;
_url: string | null;
_neighborhood: string | null;
_area: number | null;
_rooms: number | null;
_price: number | null;
_rawPrice: string | null;
_description: string| null;
_pic: string | null;
_baths: number | null;
_level: string | null;
_phone: string | null;
private constructor() {}
public static create(
id : string,
title : string,
url : string,
neighborhood: string,
area : number | string,
rooms : number | string,
price : number | string,
description : string,
pic : string,
baths : number | string,
level : string,
phone : string
) {
const house = new House();
house._id = id;
house._title = title?.trim() || "";
house._url = url?.trim() || "";
house._neighborhood = neighborhood?.trim() || "";
house._area = area ? parseInt(area.toString()) : 0;
house._rooms = rooms ? parseInt(rooms.toString()) : 0;
house._price = price ? parseInt(price.toString()?.replace(/\./g,'')) : 0;
house._rawPrice = price.toString() || "";
house._description = description?.trim() || "";
house._pic = pic?.trim() || "";
house._baths = baths ? parseInt(baths.toString()) : 0;
house._level = level?.trim() || "";
house._phone = phone?.trim() || "";
return house;
}
// Método para verificar coincidencia exacta en habitaciones, baños y precio
matchesExact(other: House): boolean {
return this.rooms === other.rooms && this.baths === other.baths && this.price === other.price;
}
public get id() {
return this._id;
}
public get title() {
return this._title;
}
public get url() {
return this._url;
}
public get neighborhood() {
return this._neighborhood;
}
public get area() {
return this._area;
}
public get rooms() {
return this._rooms;
}
public get price() {
return this._price;
}
public get rawPrice() {
return this._rawPrice;
}
public get description() {
return this._description;
}
public get pic() {
return this._pic;
}
public get baths() {
return this._baths;
}
public get level() {
return this._level;
}
public get phone() {
return this._phone;
}
}

19
server/plugins/db/init.ts Normal file
View File

@ -0,0 +1,19 @@
import { defineEventHandler } from "h3";
import connectToDatabase from "~/server/db/mysql";
import Property from "~/server/db/mysql/models/Property";
// Inicializar Sequelize con las configuraciones de Nuxt
export default defineEventHandler(async (event) => {
const sequelize = await connectToDatabase();
// Definir el modelo Property
Property(sequelize);
// Sincronizar la base de datos
try {
await sequelize?.sync({ alter: false }); // O usa { alter: true } para no perder datos existentes.
console.log("Database synchronized successfully.");
} catch (error) {
console.error("Error synchronizing database:", error);
}
});

12
server/services/cron/cron.js Normal file
View File

@ -0,0 +1,12 @@
//import cron from "node-cron";
//import fotocasaScraper from "./fotocasaScraper";
// Import other scrapers similarly
//import { notifyNewListing } from './telegramBot'; // Ensure the bot is initialized
export default function (moduleOptions) {
// Schedule scraping tasks
//cron.schedule("0 0 * * *", () => {
//fotocasaScraper();
// Call other scrapers similarly
//});
}

100
server/services/fusejs/filterProperties.js Normal file
View File

@ -0,0 +1,100 @@
// utils/filterProperties.js
import Fuse from 'fuse.js';
export const filterSimilarProperties = (properties) => {
const options = {
keys: ['_title'], // Solo buscamos similitud en el título
threshold: 0.55, // 0.6 es un umbral para 40% de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(properties, options);
const uniqueProperties = [];
const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
properties.forEach((property) => {
const propertyData = property;
if (!seen.has(propertyData._id)) {
// Inicializa el array para IDs similares en dataValues
propertyData.similarIds = [];
// Buscamos propiedades similares por título
const result = fuse.search(propertyData._title).filter((res) => {
const similarProperty = res.item;
// Filtramos resultados que no sean la misma propiedad y sean similares en título
const isSimilar = res.item._id !== propertyData._id && res.score < options.threshold;
if (isSimilar) {
propertyData.similarIds.push(similarProperty._id);
// Enriquecer la propiedad actual con datos de la propiedad similar
// propertyData.title = getLongest(propertyData.title, similarProperty.title);
// propertyData.url = propertyData.url || similarProperty.url;
// propertyData.price = propertyData.price || similarProperty.price;
// propertyData.rooms = propertyData.rooms || similarProperty.rooms;
// propertyData.area = propertyData.area || similarProperty.area;
// propertyData.level = propertyData.level || similarProperty.level;
// propertyData.description = getLongest(propertyData.description, similarProperty.description);
// propertyData.pic = propertyData.pic || similarProperty.pic;
// propertyData.baths = propertyData.baths || similarProperty.baths;
// propertyData.neighborhood = propertyData.neighborhood || similarProperty.neighborhood;
// propertyData.phone = propertyData.phone || similarProperty.phone;
// Marcar la propiedad similar como vista
//seen.add(similarProperty.id);
}
return isSimilar;
});
uniqueProperties.push(property);
}
});
return uniqueProperties;
};
export const groupSimilarProperties = (properties) => {
const filteredProperties = filterSimilarProperties(properties);
const groups = [];
const visited = new Set();
filteredProperties.forEach((property) => {
const propertyId = property._id;
if (!visited.has(propertyId)) {
const group = [property];
visited.add(propertyId);
const queue = [...property.similarIds];
while (queue.length > 0) {
const currentId = queue.shift();
if (!visited.has(currentId)) {
const similarProperty = filteredProperties.find(prop => prop._id === currentId);
if (similarProperty) {
group.push(similarProperty);
visited.add(currentId);
queue.push(...similarProperty.similarIds.filter(id => !visited.has(id)));
}
}
}
groups.push(group);
}
});
// Convert groups to desired output format
const groupedProperties = groups.reduce((acc, group, index) => {
acc[index] = group;
return acc;
}, {});
return groupedProperties;
};
const getLongest = (a, b) => {
if (!a) return b;
if (!b) return a;
return a.length >= b.length ? a : b;
};

66
server/services/fuzzySearch/fuzzySearchService.ts Normal file
View File

@ -0,0 +1,66 @@
import stringSimilarity from 'string-similarity';
export class FuzzySearchService {
_data: any = [];
_threshold: number;
private constructor(data, threshold) {
this._data = data;
this._threshold = threshold;
}
public static create(data, threshold = 0.7) {
return new FuzzySearchService(data, threshold);
}
matchSimilarProperty(property) {
const groupedData: Map<string, unknown[]> = new Map();
this.data.forEach((obj) => {
let found = false;
for (const [key, group] of groupedData.entries()) {
const representativeObj: any = group[0];
if (
this.isPropertySimilar(representativeObj[property], obj[property])
) {
group.push(obj);
groupedData.set(key, group);
found = true;
break;
}
}
if (!found) {
// Crea una nueva clave en el mapa para un nuevo grupo
groupedData.set(obj.id, [obj]);
}
});
return groupedData;
}
matchExactProperty(property) {
// Implement your exact property matching logic here
// Return the matched property
}
matchExactProperties(properties) {
// Implement your exact properties matching logic here
// Return the matched properties
}
isPropertySimilar(property, otherProperty: string): boolean {
const similarity = stringSimilarity.compareTwoStrings(property, otherProperty);
return similarity >= this.threshold;
}
get data() {
return this._data;
}
get threshold() {
return this._threshold;
}
}

21
server/services/telegram/telegramBot.js Normal file
View File

@ -0,0 +1,21 @@
import TelegramBot from 'node-telegram-bot-api';
import Listing from '../../server/db/models/Listing';
const botToken = process.env.TELEGRAM_BOT_TOKEN;
const chatId = process.env.TELEGRAM_CHAT_ID;
const bot = new TelegramBot(botToken, { polling: true });
export const notifyNewListing = async (listing) => {
const message = `
New Listing:
Title: ${listing.title}
Price: ${listing.price}
Location: ${listing.location}
Description: ${listing.description}
URL: ${listing.url}
`;
await bot.sendMessage(chatId, message);
};
export default bot;

8
server/tsconfig.json Normal file
View File

@ -0,0 +1,8 @@
{
"extends": "../.nuxt/tsconfig.server.json",
"include": [
"./node_modules/puppeteer-extra/*.d.ts",
"./node_modules/puppeteer-extra-*/*.d.ts",
"./node_modules/@types/puppeteer/index.d.ts"
]
}

10
tsconfig.json Normal file
View File

@ -0,0 +1,10 @@
{
// https://nuxt.com/docs/guide/concepts/typescript
"extends": "./.nuxt/tsconfig.json",
"compilerOptions": {
"types": [
"./types/index"
]
},
}

8
types/index.d.ts vendored Normal file
View File

@ -0,0 +1,8 @@
// types/index.d.ts
import 'nuxt'
declare module 'nuxt' {
interface Context {
$puppeteer: typeof import('puppeteer-extra')
}
}