Untitled
unknown
javascript
3 years ago
20 kB
9
Indexable
'use strict' const { parentPort, workerData } = require('worker_threads') const _ = require('lodash') const axios = require('axios').default const esb = require('elastic-builder') // the builder const geolib = require('geolib') const { stringifyUrl, getDefaultParams } = require('@property-scout/stringify-url') const { Amenity, NumberOfBed, MinLease, PropertyType, SearchSortType, Language, } = require('@property-scout/shared-enum') const { mappingAmenity } = require('../../utils/mapping-fields') const { getLogger } = require('@property-scout/logger') const { getLocaleString } = require('@property-scout/locale-strings') const logger = getLogger({ filename: __filename }) const REDUCE_PRIORITY = 0.05 function numberOfBedMapping(type) { if (!type) { return null } switch (type) { case 'one_bedroom': return NumberOfBed.OnePlus case 'two_bedrooms': return NumberOfBed.TwoPlus case 'three_bedrooms': return NumberOfBed.ThreePlus default: return NumberOfBed.FourPlus } } function toKebabCase(text) { return (text || '') .toLowerCase() .split(' ') .join('-') .split('-') .filter((c) => c) .join('-') } function pickOneEveryArray(...arr) { let combinations = [[]] for (let i = 0; i < arr.length; i++) { const reInitialize = [] const options = arr[i] combinations.forEach((comb) => { options.forEach((option) => { reInitialize.push(_.flatMap([comb, [option]])) }) }) combinations = reInitialize } return combinations } function combination(collection, n) { const array = _.values(collection) if (array.length < n) { return [] } const recur = (arr, arrLength) => { if (--arrLength < 0) { return [[]] } const combinations = [] arr = arr.slice() while (arr.length - arrLength) { const value = arr.shift() recur(arr, arrLength).forEach((comb) => { comb.unshift(value) combinations.push(comb) }) } return combinations } return recur(array, n) } function updateCombinations(combinations, key, tenure) { const currentCount = combinations.get(key) || { rents: 0, sales: 0, } if (tenure.includes('rent')) { currentCount.rents++ } if (tenure.includes('sell')) { currentCount.sales++ } combinations.set(key, currentCount) } const languages = [Language.EN, Language.TH] /** * getElasticSearchPagination * @param {*} searchUrl elasticsearch url * @param {*} bodyWithoutPagination body of query withou pagination * @param {*} limit number of item per request * @param {*} indexName index name for debug */ async function getElasticSearchPagination(searchUrl, bodyWithoutPagination, limit, indexName) { let page = 1 let resultData = [] while (true) { logger.info(`get ${indexName} page ${page}`) const bodyWithPagination = bodyWithoutPagination.size(limit).from((page - 1) * limit) page++ const { data } = await axios.post(searchUrl, bodyWithPagination) const result = data.hits.hits resultData = resultData.concat(result) if (result.length < limit) { return resultData } } } const buildingSearchUrl = `${process.env.ELASTICSEARCH_URL}/poi-building/_search` const areaSearchUrl = `${process.env.ELASTICSEARCH_URL}/poi-areas/_search` const transportationSearchUrl = `${process.env.ELASTICSEARCH_URL}/poi-transportation/_search` const areaFields = ['id', 'coordinates', 'name', 'type_short.en'] const buildingFields = ['id', 'lat', 'lng', 'name', 'building_type'] const transportationFields = ['id', 'gps_lat', 'gps_long', 'transportation_system', 'station'] const buildingBody = esb .requestBodySearch() .size(5000) .query( esb .boolQuery() .must([esb.existsQuery('lat'), esb.existsQuery('lng'), esb.existsQuery('building_type')]) .mustNot(esb.matchQuery('priority', ['DUPLICATE_URL_STRING', 'DISREGARD'].join(' '))), ) .source(buildingFields) .sorts([esb.sort('id', 'asc')]) const areaBody = esb .requestBodySearch() .source(areaFields) .query( esb .boolQuery() .mustNot(esb.matchQuery('priority', 'DISREGARD')) .must([esb.existsQuery('coordinates')]), ) .sorts([esb.sort('id', 'asc')]) const transportationBody = esb .requestBodySearch() .source(transportationFields) .query(esb.boolQuery().must([esb.existsQuery('gps_lat'), esb.existsQuery('gps_long')])) .sorts([esb.sort('id', 'asc')]) const usingBedrooms = [NumberOfBed.OnePlus, NumberOfBed.TwoPlus, NumberOfBed.ThreePlus, NumberOfBed.FourPlus] const usingMinLeases = [ MinLease.Monthly, MinLease.TwoMonths, MinLease.ThreeMonths, MinLease.SixMonths, MinLease.TwelveMonths, ] const usingPropertyTypes = [ PropertyType.Apartment, PropertyType.ServicedApartment, PropertyType.Condo, PropertyType.House, PropertyType.Townhouse, ] const usingAmenities = [ Amenity.Balcony, Amenity.PetFriendly, Amenity.Gym, Amenity.Parking, Amenity.Pool, Amenity.Oven, Amenity.Playground, Amenity.Garden, ] /* mapping amenities (use in client) to correct amenities and add it to the selectedFields */ const amenityMap = usingAmenities.map((amenity) => ({ key: amenity, value: mappingAmenity(amenity), })) const mappingAmenities = _.flatMap(amenityMap, (item) => item.value) const rentalFields = [ 'id', 'propertyType', 'numberBedrooms', 'minLeasePeriod', // 'gpsLong', 'gpsLat', 'buildingId.name_en', 'buildingId.name_th', 'buildingId.building_id', 'buildingId.building_type', 'province_ps_en', 'district_ps_en', 'subdistrict_ps_en', 'neighborhood_ps_en', 'tenure', ...mappingAmenities, ] const serpTypes = ['rents', 'sales'] async function handler() { const MAX_AMENITIES_SELECTED = 3 const TRANSPORTATION_DEFAULT_RADIUS = 750 // 750 meters const BUILDING_DEFAULT_RADIUS = 500 // 500 meters const buildings = await getElasticSearchPagination(buildingSearchUrl, buildingBody, 1000, 'poi-building') logger.info(`Get total ${buildings.length} buildings`) const buildingIdsSet = new Set(buildings.map((building) => building._source.id)) const areas = await getElasticSearchPagination(areaSearchUrl, areaBody, 1000, 'poi-areas') logger.info(`Get total ${areas.length} areas`) // get all transportation from elasticsearch const transportations = await getElasticSearchPagination( transportationSearchUrl, transportationBody, 1000, 'poi-transportations', ) logger.info(`Get total ${transportations.length} transportations`) // get rental from elasticsearch const limit = 1000 let page = 1 const rentalSearchUrl = `${process.env.ELASTICSEARCH_URL}/rental/_search` /** * Combinations with rental count for rent and sale */ const combinations = new Map() const buildingNameMap = new Map() const transportationNameMap = new Map() const areaNameMap = new Map() const totalRentals = { rents: 0, sales: 0, } while (true) { try { // get rental with pagination const bodyPagination = esb .requestBodySearch() .size(limit) .from((page - 1) * limit) .source(rentalFields) .query( esb .boolQuery() .must([ esb.matchQuery('status', 'ACTIVE'), esb.existsQuery('minLeasePeriod'), esb.termQuery('minLeasePeriod', usingMinLeases), esb.existsQuery('propertyType'), esb.termQuery('propertyType', usingPropertyTypes), esb.termsQuery('tenure', ['rentsell', 'sell']), ]), ) .sorts([esb.sort('id', 'asc')]) const { data } = await axios.post(rentalSearchUrl, bodyPagination) const rentals = data.hits.hits if (!rentals.length) { break } rentals.forEach(({ _source: rental }) => { const poiNames = [] if (rental.tenure.includes('rent')) { totalRentals.rents++ } if (rental.tenure.includes('sell')) { totalRentals.sales++ } const arrPickOne = [] const { propertyType, numberBedrooms, gpsLat, gpsLong, minLeasePeriod, buildingId } = rental arrPickOne.push([`PT:${propertyType}`]) const bedroomConverted = numberOfBedMapping(numberBedrooms) const bedroomIndex = usingBedrooms.indexOf(bedroomConverted) if (bedroomIndex > -1) { const rentalBedrooms = usingBedrooms.slice(0, bedroomIndex + 1) arrPickOne.push(rentalBedrooms.map((rentalBedroom) => `NB:${rentalBedroom}`)) } const minLeaseIndex = usingMinLeases.indexOf(minLeasePeriod) const rentalMinLeases = usingMinLeases .slice(minLeaseIndex, usingMinLeases.length) .map((rentalMinLease) => `ML:${rentalMinLease}`) if (_.isNumber(gpsLat) && _.isNumber(gpsLong)) { // check rental in area (polygon) const rentalLocation = { latitude: gpsLat, longitude: gpsLong, } areas.forEach(({ _source: area }) => { let inPolygon = false const { type, name } = area switch (type) { case 'province': if (name.en === rental.province_ps_en) { inPolygon = true } break case 'district': if (name.en === rental.district_ps_en) { inPolygon = true } break case 'subdistrict': if (name.en === rental.subdistrict_ps_en) { inPolygon = true } break case 'neighborhood': if (name.en === rental.neighborhood_ps_en) { inPolygon = true } break default: break } if (inPolygon) { // pois.push(`AID:${area.id}`); const areaNameEn = toKebabCase(area.name.en) areaNameMap.set(areaNameEn, { id: area.id, name: area.name }) poiNames.push(`AN:${areaNameEn}`) } }) /* - find transportation with radius 1.5km */ transportations.forEach(({ _source }) => { const { id, station, gps_lat, gps_long, transportation_system: transportationSystem } = _source const withinRadius = geolib.isPointWithinRadius( rentalLocation, { latitude: gps_lat, longitude: gps_long, }, TRANSPORTATION_DEFAULT_RADIUS, ) if (withinRadius) { // transportationMap.set(id, { name: station, transportationSystem }); // pois.push(`TID:${id}`); const transportationNameEn = toKebabCase(station.en) transportationNameMap.set(transportationNameEn, { id, transportationSystem, name: station }) poiNames.push(`TN:${transportationNameEn}`) } }) if (workerData.flag === 'radius') { // check building with radius 0.5km buildings.forEach(({ _source }) => { const { id, lat, lng, name, building_type } = _source const withinRadius = geolib.isPointWithinRadius( rentalLocation, { latitude: lat, longitude: lng, }, BUILDING_DEFAULT_RADIUS, ) if (withinRadius) { // buildingMap.set(id, { name }); // pois.push(`BID:${id}`); const buildingNameEn = toKebabCase(name.en) buildingNameMap.set(buildingNameEn, { id, name, building_type }) poiNames.push(`BN:${buildingNameEn}`) } }) } else { if (buildingId && buildingId.building_id && buildingIdsSet.has(Number(buildingId.building_id))) { // buildingMap.set(Number(buildingId.building_id), { // name: { // en: buildingId.name_en, // th: buildingId.name_th, // }, // }); const buildingNameEn = toKebabCase(buildingId.name_en) buildingNameMap.set(buildingNameEn, { id: Number(buildingId.building_id), name: { en: buildingId.name_en, th: buildingId.name_th, }, building_type: buildingId.building_type, }) // pois.push(`BID:${buildingId.building_id}`); poiNames.push(`BN:${buildingNameEn}`) } } } if (poiNames.length) { arrPickOne.push(poiNames) } let convertMappings = [] // - convert rental amenity to original mappingAmenities.forEach((amenity) => { if (rental[amenity]) { const convertMapping = amenityMap.find((a) => a.value.includes(amenity)).key convertMappings.push(convertMapping) } }) convertMappings = _.uniq(convertMappings).sort() let amenityCombinations = [] // - create serps with amenity only if (convertMappings.length) { amenityCombinations = _.flatMap( _.times(_.min([convertMappings.length, MAX_AMENITIES_SELECTED]), (num) => { return combination(convertMappings, num + 1) }), ) } if (amenityCombinations.length) { arrPickOne.push( amenityCombinations.map((amenityCombination) => { const sortedAmenities = _.sortBy(amenityCombination, (amenity) => amenity.toLowerCase().replace(/^amenity/, ''), ) return `AM:${sortedAmenities.join(',')}` }), ) } const propsToPick = _.flatMap( _.times(arrPickOne.length, (num) => { return combination(arrPickOne, num + 1) }), ) const combinationsArr = _.flatMap( propsToPick.map((propToPick) => { // by default, all serp always have a min_lease const withMinleaseProps = rentalMinLeases.length ? [...propToPick, rentalMinLeases] : propToPick return pickOneEveryArray(...withMinleaseProps) }), ) combinationsArr.forEach((combinationArr) => { updateCombinations(combinations, combinationArr.join('|'), rental.tenure) }) }) logger.info(`Get rental page ${page}`) page++ } catch (error) { logger.error(error) } } // combination for base serp url combinations.set('', totalRentals) generateSerpsUrl({ combinations, buildingNameMap, areaNameMap, transportationNameMap, }) } function generateSerpsUrl({ combinations, buildingNameMap, areaNameMap, transportationNameMap }) { const entries = combinations.entries() const sortTypes = [ SearchSortType.Newest, SearchSortType.PriceLow, SearchSortType.PriceHigh, SearchSortType.Recommended, ] const defaultParams = getDefaultParams() const priorityGroup = Object.fromEntries(_.times(21, (num) => [Number((num * REDUCE_PRIORITY).toFixed(2)), []])) const totalCombinations = combinations.size let idx = 0 while (idx < totalCombinations) { const { value } = entries.next() const [serpString, totalRentals] = value const serpObj = {} const separations = serpString.split('|') separations.forEach((separation) => { /* * FS-1051 ... The error occurs when there is a `:` character in the value * separation = prefix join value by `:` character like `prefix:value` * when I split string with `:` character * the result at index 0 should be prefix * the rest after joined with `:` character should be value * ex: poi building (3620): Condo Wassana City:Building B * `separation` will be `BN:condo-wassana-city:building-b` * `parts` will be ["BN", "condo-wassana-city", "building-b"] * `prefix` will be `BN` * `value` will be `condo-wassana-city:building-b` (["condo-wassana-city", "building-b"].join(':')) */ const parts = separation.split(':') const prefix = parts[0] const val = parts.slice(1).join(':') switch (prefix) { case 'PT': // propertyType serpObj.propertyType = [val] break case 'NB': // numberBedrooms serpObj.numberBedrooms = val break case 'AM': // amenities serpObj.amenities = val ? val.split(',') : [] break case 'BN': // building name const buildingMapValue = buildingNameMap.get(val) serpObj.buildingId = buildingMapValue.id serpObj.where = buildingMapValue.name // default value for building_type is condo serpObj.building_type = buildingMapValue.building_type || 'condo' break case 'AN': // area name const areaMapValue = areaNameMap.get(val) serpObj.areaId = areaMapValue.id serpObj.where = areaMapValue.name break case 'TN': // transportation name const transportationMapValue = transportationNameMap.get(val) serpObj.transportationId = transportationMapValue.id serpObj.where = transportationMapValue.name serpObj.transportationSystem = transportationMapValue.transportationSystem break case 'ML': // minLease serpObj.minLease = val break default: break } }) for (let index = 0; index < sortTypes.length; index++) { const sortBy = sortTypes[index] const enParams = { ...defaultParams, ...serpObj, where: serpObj.where?.en, sortBy, } const shouldFollowing = getFollowing(enParams) if (!shouldFollowing) { continue } const priority = getPriority(enParams) serpTypes.forEach((serpType) => { const [enUrl, thUrl] = languages.map( (lang) => stringifyUrl({ request: { ...defaultParams, ...serpObj, where: serpObj.where ? serpObj.where[lang] : undefined, sortBy, serp_type: serpType, }, t: (key) => getLocaleString(lang, 'common', key), }).path, ) priorityGroup[priority].push({ enUrl, thUrl, totalRentals: totalRentals[serpType], }) }) } idx++ } const priorityKeys = _.sortBy( _.keys(priorityGroup).filter((k) => priorityGroup[k].length), (key) => -Number(key), ) parentPort.postMessage({ serpsPriorityGroup: Object.fromEntries(priorityKeys.map((key) => [key, priorityGroup[key]])), }) } function getPriority(params) { let level = 0 if (params.propertyType && params.propertyType.length) { level++ } if (params.amenities && params.amenities.length) { level += params.amenities.length } if (params.minLease && params.minLease !== MinLease.TwelveMonths) { level++ } if (params.numberBedrooms && params.numberBedrooms !== NumberOfBed.AnyType) { level++ } if (params.sortBy) { level++ } return Number((1 - REDUCE_PRIORITY * level).toFixed(2)) } /** * FS-1240 Sitemap / remove building POI SERPS with filters amenity, property type, min. lease term, sorting * @returns */ function getFollowing(params) { if (params.buildingId) { const withPropertyTypeFilter = !!(params.propertyType && params.propertyType.length) const withAmenitiesFilter = !!(params.amenities && params.amenities.length) const withNondefaultSorting = !!(params.sortBy && params.sortBy !== SearchSortType.Recommended) const withNonDefaultMinlease = !!(params.minLease && params.minLease !== MinLease.TwelveMonths) return ![withPropertyTypeFilter, withAmenitiesFilter, withNondefaultSorting, withNonDefaultMinlease].some(Boolean) } return true } if (require.main === module) { handler() }
Editor is loading...