alexandrump/hometify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Delete some unused stuff

Browse Source
This commit is contained in:
Alejandro Martínez 2024-10-01 19:49:44 +02:00
parent b4c2d92c17
commit e76e558a7a
20 changed files with 0 additions and 698 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
bak/baxk1.js
View File

@ -1,84 +0,0 @@
// utils/filterProperties.js
import Fuse from 'fuse.js';
export const filterSimilarProperties = (properties) => {
const options = {
keys: ['dataValues.title'], // Buscamos similitud en el título dentro de dataValues
threshold: 0.6, // 0.6 es un umbral para 40% de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(properties, options);
const uniqueProperties = [];
const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
properties.forEach((property) => {
const propertyData = property.dataValues;
if (!seen.has(propertyData.id)) {
// Inicializa el array para IDs similares en dataValues
propertyData.similarIds = [];
// Buscamos propiedades similares por título
const result = fuse.search(propertyData.title).filter((res) => {
const similarProperty = res.item.dataValues;
// Filtramos resultados que no sean la misma propiedad y sean similares en título
const isSimilar = res.item.dataValues.id !== propertyData.id && res.score < options.threshold;
if (isSimilar) {
propertyData.similarIds.push(similarProperty.id);
// Enriquecer la propiedad actual con datos de la propiedad similar
propertyData.title = getLongest(propertyData.title, similarProperty.title);
propertyData.url = propertyData.url || similarProperty.url;
propertyData.price = propertyData.price || similarProperty.price;
propertyData.rooms = propertyData.rooms || similarProperty.rooms;
propertyData.area = propertyData.area || similarProperty.area;
propertyData.level = propertyData.level || similarProperty.level;
propertyData.description = getLongest(propertyData.description, similarProperty.description);
propertyData.pic = propertyData.pic || similarProperty.pic;
propertyData.baths = propertyData.baths || similarProperty.baths;
propertyData.neighborhood = propertyData.neighborhood || similarProperty.neighborhood;
propertyData.phone = propertyData.phone || similarProperty.phone;
// Marcar la propiedad similar como vista
seen.add(similarProperty.id);
}
return isSimilar;
});
// Comprobamos coincidencia exacta en price, rooms y area
const matchingProperties = result.filter((res) => {
const similarProperty = res.item.dataValues;
return (
similarProperty.price === propertyData.price &&
(similarProperty.rooms === propertyData.rooms ||
similarProperty.area === propertyData.area)
);
});
if (matchingProperties.length === 0) {
uniqueProperties.push(property);
}
// Añadir los IDs de propiedades similares al array similarIds y marcarlas como vistas
matchingProperties.forEach((res) => {
const similarProperty = res.item.dataValues;
propertyData.similarIds.push(similarProperty.id);
seen.add(similarProperty.id);
});
// Añadir la propiedad actual como vista
seen.add(propertyData.id);
}
});
return uniqueProperties;
};
const getLongest = (a, b) => {
if (!a) return b;
if (!b) return a;
return a.length >= b.length ? a : b;
};

57
bak/completePropertyData1.js
View File

@ -1,57 +0,0 @@
// utils/completePropertyData.js
import Fuse from 'fuse.js';
export const completePropertyData = (uniqueProperties, allProperties) => {
// Creamos una copia de allProperties con sólo los dataValues
const allDataValues = allProperties.map(prop => prop.dataValues);
const options = {
keys: ['title'], // Buscamos similitud en el título
threshold: 0.6, // Umbral de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(allDataValues, options);
const enrichedProperties = uniqueProperties.map(property => {
// Buscamos propiedades similares por título para completar datos
const result = fuse.search(property.title).filter((res) => {
// Filtramos resultados que no sean la misma propiedad y sean similares en título
return res.item.id !== property.id && res.score < options.threshold;
});
// Combinamos las propiedades similares para completar datos
let enrichedProperty = { ...property };
result.forEach((res) => {
const item = res.item;
enrichedProperty = {
...enrichedProperty,
title: getLongest(enrichedProperty.title, item.title),
url: enrichedProperty.url || item.url,
price: enrichedProperty.price || item.price,
rooms: enrichedProperty.rooms || item.rooms,
area: enrichedProperty.area || item.area,
level: enrichedProperty.level || item.level,
description: getLongest(enrichedProperty.description, item.description),
pic: enrichedProperty.pic || item.pic,
baths: enrichedProperty.baths || item.baths,
neighborhood: enrichedProperty.neighborhood || item.neighborhood,
phone: enrichedProperty.phone || item.phone,
createdAt: enrichedProperty.createdAt || item.createdAt,
updatedAt: enrichedProperty.updatedAt || item.updatedAt
};
});
return enrichedProperty;
});
return enrichedProperties;
};
// Función auxiliar para obtener el texto más largo o el que no sea nulo
const getLongest = (a, b) => {
if (!a) return b;
if (!b) return a;
return a.length >= b.length ? a : b;
};

62
bak/filterProperties copy.js
View File

@ -1,62 +0,0 @@
// utils/filterProperties.js
import Fuse from 'fuse.js';
export const filterSimilarProperties = (properties) => {
const options = {
keys: ['title'], // Solo buscamos similitud en el título
threshold: 0.6, // 0.6 es un umbral para 40% de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(properties, options);
const uniqueProperties = [];
const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
properties.forEach((property) => {
const propertyData = property.dataValues;
if (!seen.has(propertyData.id)) {
// Inicializa el array para IDs similares en dataValues
propertyData.similarIds = [];
// Buscamos propiedades similares por título
const result = fuse.search(propertyData.title).filter((res) => {
const similarProperty = res.item.dataValues;
// Filtramos resultados que no sean la misma propiedad y sean similares en título
const isSimilar = res.item.dataValues.id !== propertyData.id && res.score < options.threshold;
if (isSimilar) {
propertyData.similarIds.push(similarProperty.id);
// Enriquecer la propiedad actual con datos de la propiedad similar
// propertyData.title = getLongest(propertyData.title, similarProperty.title);
// propertyData.url = propertyData.url || similarProperty.url;
// propertyData.price = propertyData.price || similarProperty.price;
// propertyData.rooms = propertyData.rooms || similarProperty.rooms;
// propertyData.area = propertyData.area || similarProperty.area;
// propertyData.level = propertyData.level || similarProperty.level;
// propertyData.description = getLongest(propertyData.description, similarProperty.description);
// propertyData.pic = propertyData.pic || similarProperty.pic;
// propertyData.baths = propertyData.baths || similarProperty.baths;
// propertyData.neighborhood = propertyData.neighborhood || similarProperty.neighborhood;
// propertyData.phone = propertyData.phone || similarProperty.phone;
// Marcar la propiedad similar como vista
//seen.add(similarProperty.id);
}
return isSimilar;
});
uniqueProperties.push(property);
}
});
return uniqueProperties;
};
const getLongest = (a, b) => {
if (!a) return b;
if (!b) return a;
return a.length >= b.length ? a : b;
};

48
bak/filterProperties1.js
View File

@ -1,48 +0,0 @@
// utils/filterProperties.js
import Fuse from 'fuse.js';
export const filterSimilarProperties = (properties) => {
const options = {
keys: ['title'], // Solo buscamos similitud en el título
threshold: 0.6, // 0.6 es un umbral para 40% de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(properties, options);
const uniqueProperties = [];
const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
properties.forEach((property) => {
if (!seen.has(property.id)) {
// Buscamos propiedades similares por título
const result = fuse.search(property.title).filter((res) => {
// Filtramos resultados que no sean la misma propiedad y sean similares en título
return res.item.id !== property.id && res.score < options.threshold;
});
// Comprobamos coincidencia exacta en price, rooms y area
const matchingProperties = result.filter((res) => {
const item = res.item;
return (
item.price === property.price &&
(item.rooms === property.rooms ||
item.area === property.area)
);
});
if (matchingProperties.length === 0) {
uniqueProperties.push(property);
}
// Marcar propiedades similares como vistas
matchingProperties.forEach((res) => {
seen.add(res.item.id);
});
// Añadir la propiedad actual como vista
seen.add(property.id);
}
});
return uniqueProperties;
};

48
bak/filterProperties_bak.js
View File

@ -1,48 +0,0 @@
// utils/filterProperties.js
import Fuse from 'fuse.js';
export const filterSimilarProperties = (properties) => {
const options = {
keys: ['title'], // Solo buscamos similitud en el título
threshold: 0.6, // 0.6 es un umbral para 40% de similitud
includeScore: true // Incluye la puntuación de similitud
};
const fuse = new Fuse(properties, options);
const uniqueProperties = [];
const seen = new Set(); // Usaremos un conjunto para rastrear propiedades vistas
properties.forEach((property) => {
if (!seen.has(property.id)) {
// Buscamos propiedades similares por título
const result = fuse.search(property.title).filter((res) => {
// Filtramos resultados que no sean la misma propiedad y sean similares en título
return res.item.id !== property.id && res.score < options.threshold;
});
// Comprobamos coincidencia exacta en price, rooms y area
const matchingProperties = result.filter((res) => {
const item = res.item;
return (
item.price === property.price &&
(item.rooms === property.rooms ||
item.area === property.area)
);
});
if (matchingProperties.length === 0) {
uniqueProperties.push(property);
}
// Marcar propiedades similares como vistas
matchingProperties.forEach((res) => {
seen.add(res.item.id);
});
// Añadir la propiedad actual como vista
seen.add(property.id);
}
});
return uniqueProperties;
};

56
bak/fotocasaScraper.js
View File

@ -1,56 +0,0 @@
import puppeteer from 'puppeteer';
import Listing from '../../server/db/models/Listing';
//import { notifyNewListing } from './telegramBot';
export default async function scrapeFotocasa() {
console.log('Starting Fotocasa scraping process');
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
const page = await browser.newPage();
await page.goto('https://www.fotocasa.es/es/comprar/viviendas/granada-capital/todas-las-zonas/l');
const pageSourceHTML = await page.content();
await browser.close();
console.log('Navigated to Fotocasa',pageSourceHTML);
try {
const listings = await page.evaluate(() => {
const results = [];
document.querySelectorAll('.re-CardPackPremium').forEach((element) => {
const title = element.querySelector('.re-CardTitle')?.innerText.trim();
const priceText = element.querySelector('.re-CardPrice')?.innerText.trim();
const price = parseFloat(priceText.replace(/[^\d]/g, ''));
const location = title;
const description = element.querySelector('.re-CardDescription-text')?.innerText.trim();
const url = element.querySelector('a')?.href;
if (title && price && location && description && url) {
results.push({ title, price, location, description, url });
}
console.log(title, price, location, description, url);
});
return results;
});
console.log(`Found ${listings.length} listings`);
for (const listingData of listings) {
try {
const listing = new Listing(listingData);
//await listing.save();
console.log(`Saved listing to database: ${listing.title}`);
//await notifyNewListing(listing);
console.log(`Sent notification for listing: ${listing.title}`);
} catch (innerError) {
console.error(`Error processing listing: ${innerError.message}`);
}
}
} catch (error) {
console.error('Error scraping Fotocasa:', error.message);
throw new Error('Failed to scrape Fotocasa');
} finally {
await browser.close();
console.log('Finished Fotocasa scraping process');
}
}

76
bak/habitacliaScraper.js
View File

@ -1,76 +0,0 @@
//import { notifyNewListing } from './telegramBot';
// import Listing from '../../server/db/models/Listing';
export default async function scrapeHabitaclia(puppeteer) {
console.log("Starting Habitaclia scraping process");
const url = 'https://idealista7.p.rapidapi.com/listhomes?order=relevance&operation=sale&locationId=0-EU-ES-28-07-001-079&locationName=Madrid&numPage=1&maxItems=40&location=es&locale=en';
const options = {
method: 'GET',
headers: {
'x-rapidapi-key': '080e9362e4mshdbcf5473f82adb9p196205jsne6a02c4e6381',
'x-rapidapi-host': 'idealista7.p.rapidapi.com'
}
};
try {
const response = await fetch(url, options);
const result = await response.text();
console.log(result);
} catch (error) {
console.error(error);
}
// const browser = await puppeteer.launch({
// headless: true,
// args: [
// '--no-sandbox' ,
// '--disable-setuid-sandbox'
// ]
// });
// const page = await browser.newPage();
// console.log(puppeteer, browser);
try {
// await page.goto('https://www.habitaclia.com/viviendas-granada_ciudad.htm', {
// waitUntil: 'networkidle2',
// });
// console.log('Navigated to Habitaclia');
// const content = await page.content();
// console.log(content);
// await page.screenshot({ path: 'habitaclia_state.png', fullPage: true })
// const listings = await page.evaluate(() => {
// const results = [];
// document.querySelectorAll('.list-item').forEach((element) => {
// const title = element.querySelector('.list-item-title')?.innerText.trim();
// const priceText = element.querySelector('.list-item-price')?.innerText.trim();
// const price = parseFloat(priceText.replace(/[^\d]/g, ''));
// const location = element.querySelector('.list-item-location')?.innerText.trim();
// const description = element.querySelector('.list-item-description')?.innerText.trim();
// const url = element.querySelector('.list-item-title a')?.href;
// if (title && price && location && description && url) {
// results.push({ title, price, location, description, url });
// }
// });
// return results;
// });
// console.log(`Found ${listings.length} listings`);
// for (const listingData of listings) {
// try {
// const listing = new Listing(listingData);
// //await listing.save();
// console.log(`Saved listing to database: ${listing.title}`);
// //await notifyNewListing(listing);
// console.log(`Sent notification for listing: ${listing.title}`);
// } catch (innerError) {
// console.error(`Error processing listing: ${innerError.message}`);
// }
// }
} catch (error) {
console.error("Error scraping Habitaclia:", error.message);
throw new Error("Failed to scrape Habitaclia");
} finally {
await browser.close();
console.log("Finished Habitaclia scraping process");
}
}

43
bak/index.cjs
View File

@ -1,43 +0,0 @@
'use strict';
const fs = require('fs');
const path = require('path');
const Sequelize = require('sequelize');
const process = require('process');
const basename = path.basename(__filename);
const env = process.env.NODE_ENV || 'development';
const config = require(__dirname + '/../../../../config/sequelize.js')[env];
const db = {};
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(config.database, config.username, config.password, config);
}
fs
.readdirSync(__dirname)
.filter(file => {
return (
file.indexOf('.') !== 0 &&
file !== basename &&
file.slice(-3) === '.js' &&
file.indexOf('.test.js') === -1
);
})
.forEach(file => {
const model = require(path.join(__dirname, file))(sequelize, Sequelize.DataTypes);
db[model.name] = model;
});
Object.keys(db).forEach(modelName => {
if (db[modelName].associate) {
db[modelName].associate(db);
}
});
db.sequelize = sequelize;
db.Sequelize = Sequelize;
module.exports = db;

5
bak/listings.ts
View File

@ -1,5 +0,0 @@
// import Listing from '../db/models/Listing';
export default defineEventHandler(async () => {
// return await Listing.find();
});

68
bak/nitroPlugin.ts
View File

@ -1,68 +0,0 @@
/* import {DEFAULT_INTERCEPT_RESOLUTION_PRIORITY} from 'puppeteer';
import Puppeteer from 'puppeteer';
import { addExtra } from 'puppeteer-extra';
import StealthPlugin from "puppeteer-extra-plugin-stealth";
import AdblockerPlugin from "puppeteer-extra-plugin-adblocker";
import chromeApp from "puppeteer-extra-plugin-stealth/evasions/chrome.app";
import chromeCsi from "puppeteer-extra-plugin-stealth/evasions/chrome.csi";
import chromeLoadTimes from "puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes";
import chromeRuntime from "puppeteer-extra-plugin-stealth/evasions/chrome.runtime";
import defaultArgs from "puppeteer-extra-plugin-stealth/evasions/defaultArgs";
import iframeContentWindow from "puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow";
import mediaCodecs from "puppeteer-extra-plugin-stealth/evasions/media.codecs";
import navigatorHardwareConcurrency from "puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency";
import navigatorLanguages from "puppeteer-extra-plugin-stealth/evasions/navigator.languages";
import navigatorPermissions from "puppeteer-extra-plugin-stealth/evasions/navigator.permissions";
import navigatorPlugins from "puppeteer-extra-plugin-stealth/evasions/navigator.plugins";
import navigatorVendor from "puppeteer-extra-plugin-stealth/evasions/navigator.vendor";
import navigatorWebdriver from "puppeteer-extra-plugin-stealth/evasions/navigator.webdriver";
import sourceUrl from "puppeteer-extra-plugin-stealth/evasions/sourceurl";
import userAgentOverride from "puppeteer-extra-plugin-stealth/evasions/user-agent-override";
import webglVendor from "puppeteer-extra-plugin-stealth/evasions/webgl.vendor";
import windowOuterDimensions from "puppeteer-extra-plugin-stealth/evasions/window.outerdimensions";
export default defineNitroPlugin((nitroApp) => {
console.log("Loading puppeteer plugin...");
const puppeteer = addExtra(Puppeteer);
const stealth = StealthPlugin();
configureEvasions(stealth);
puppeteer.use(stealth);
puppeteer.use(
AdblockerPlugin({
// Optionally enable Cooperative Mode for several request interceptors
interceptResolutionPriority: DEFAULT_INTERCEPT_RESOLUTION_PRIORITY
})
)
nitroApp.hooks.hook("request", (event) => {
event.context.$puppeteer = puppeteer;
});
});
// Configurar evasiones
const configureEvasions = (stealth) => {
// Clear existing evasions
stealth.enabledEvasions.clear();
// Add evasions explicitly
stealth.enabledEvasions.add(chromeApp);
stealth.enabledEvasions.add(chromeCsi);
stealth.enabledEvasions.add(chromeLoadTimes);
stealth.enabledEvasions.add(chromeRuntime);
stealth.enabledEvasions.add(defaultArgs);
stealth.enabledEvasions.add(iframeContentWindow);
stealth.enabledEvasions.add(mediaCodecs);
stealth.enabledEvasions.add(navigatorHardwareConcurrency);
stealth.enabledEvasions.add(navigatorLanguages);
stealth.enabledEvasions.add(navigatorPermissions);
stealth.enabledEvasions.add(navigatorPlugins);
stealth.enabledEvasions.add(navigatorVendor);
stealth.enabledEvasions.add(navigatorWebdriver);
stealth.enabledEvasions.add(sourceUrl);
stealth.enabledEvasions.add(userAgentOverride);
stealth.enabledEvasions.add(webglVendor);
stealth.enabledEvasions.add(windowOuterDimensions);
}; */

55
bak/pisoscomScraper.js
View File

@ -1,55 +0,0 @@
import puppeteer from 'puppeteer';
import Listing from '../../server/db/models/Listing';
//import { notifyNewListing } from './telegramBot';
export default async function scrapePisoscom() {
console.log('Starting Pisos.com scraping process');
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
const page = await browser.newPage();
try {
await page.goto('https://www.pisos.com/venta/pisos-area_de_granada_granada_capital/', {
waitUntil: 'networkidle2',
});
console.log('Navigated to Pisos.com');
const listings = await page.evaluate(() => {
const results = [];
document.querySelectorAll('.ad-preview').forEach((element) => {
const title = element.querySelector('.ad-preview__title').innerText.trim();
const priceText = element.querySelector('.ad-preview__price').innerText.trim();
const price = parseFloat(priceText.replace(/[^\d]/g, ''));
const location = element.querySelector('.ad-preview__subtitle').innerText.trim();
const description = element.querySelector('.ad-preview__description').innerText.trim();
const phone = element.querySelector('.contact-box__phone').getAttribute('data-number');
const url = element.getAttribute('data-lnk-href');
if (title && price && location && description && url) {
results.push({ title, price, location, description, url, phone });
}
});
return results;
});
console.log(`Found ${listings.length} listings. Listing data: ${JSON.stringify(listings)}`);
for (const listingData of listings) {
try {
const listing = new Listing(listingData);
//await listing.save();
console.log(`Saved listing to database: ${listing.title}`);
//await notifyNewListing(listing);
console.log(`Sent notification for listing: ${listing.title}`);
} catch (innerError) {
console.error(`Error processing listing: ${innerError.message}`);
}
}
} catch (error) {
console.error('Error scraping Pisos.com:', error.message);
throw new Error('Failed to scrape Pisos.com');
} finally {
await browser.close();
console.log('Finished Pisos.com scraping process');
}
}

51
bak/scrape.vue
View File

@ -1,51 +0,0 @@
<template>
<div class="flex items-center justify-center min-h-screen bg-gray-100">
<div class="bg-white p-6 rounded-lg shadow-lg text-center max-w-md w-full">
<h1 class="text-2xl font-bold mb-4">Initiate Scraping</h1>
<button
@click="scrape"
class="bg-blue-500 text-white font-semibold py-2 px-4 rounded hover:bg-blue-700 transition-colors duration-300"
>
Start Scraping
</button>
<p v-if="message" class="mt-4 text-green-600">{{ message }}</p>
</div>
</div>
</template>
<script setup>
import { ref } from 'vue';
const message = ref('');
const scrapers= [
//'scrapepisoscom',
'scrapehabitaclia'
]
const scrape = async () => {
message.value = 'Starting scraping...\n';
for (const portal of scrapers) {
await initiateScraping(portal);
}
};
const initiateScraping = async (portal) => {
try {
const response = await fetch(`/api/${portal}`, {
method: 'POST',
});
const result = await response.json();
if (result.success) {
message.value = 'Scraping initiated successfully: ' + JSON.stringify(result);
} else {
message.value = `Error: ${result.message}`;
}
} catch (error) {
message.value = `Error: ${error.message}`;
}
};
</script>
<style>
/* Tailwind CSS se encargará de los estilos, por lo que no se necesita nada aquí */
</style>

11
bak/scrapehabitaclia.js
View File

@ -1,11 +0,0 @@
import scrapeHabitaclia from '../services/scraper/habitacliaScraper';
export default defineEventHandler(async (event) => {
const puppeteer = event.context.$puppeteer;
try {
const result = await scrapeHabitaclia(puppeteer);
return { success: true, message: 'Habitacia scraping initiated.' , data: result || {}};
} catch (error) {
return { success: false, message: 'Failed to initiate Habitacia scraping.', error: error.message };
}
});

10
bak/scrapepisoscom.js
View File

@ -1,10 +0,0 @@
// import pisoscomScraper from '../../modules/scraping/pisoscomScraper';
export default defineEventHandler(async (event) => {
try {
const result = await pisoscomScraper();
return { success: true, message: 'Fotocasa scraping initiated.' , data: result || {}};
} catch (error) {
return { success: false, message: 'Failed to initiate Fotocasa scraping.', error: error.message };
}
});

15
bak/test.ts
View File

@ -1,15 +0,0 @@
import { createClient } from 'pexels';
const client = createClient('dKrFwkztM2MT9ss60BIBukRyTQpwEN4cFdydJis2zi2xeQ1NC4irqWGl');
export default defineEventHandler(async (event) => {
const {query} = await readBody(event);
const result = await client.photos.search({ query, per_page: 20 });
return result?.photos.map(item => {
return {
id :item.id,
alt: item.alt,
src: item.src.large
};
});
});

9
bak/useDatabase.js
View File

@ -1,9 +0,0 @@
import mongoose from 'mongoose';
export const useDatabase = () => {
const config = useRuntimeConfig();
mongoose.connect(config.mongodbUri, { useNewUrlParser: true, useUnifiedTopology: true });
return mongoose;
}

0
composables/.gitikeep Normal file
View File

0
modules/.gitikeep Normal file
View File

0
plugins/.gitikeep Normal file
View File

BIN
project.zip
View File

Binary file not shown.