tnrastertile.js 11.78 KiB
const express = require('express');
const mbgl = require('@mapbox/mapbox-gl-native');
const sharp = require('sharp');
const request = require('request');
const SphericalMercator = require('@mapbox/sphericalmercator');
const fs = require('fs');
const zlib = require('zlib');
const path = require("path");
const crc = require('crc');
const async = require('async');
const advancedPool = require('advanced-pool');
const renderVectorTile = require('./tnvectortile');
const s3 = require('s3');
const aws = require('aws-sdk');
const tileSize = 256;
const merc = new SphericalMercator();
const SCALE_PATTERN = '@[234]x';
mbgl.on('message', function(e) {
  if (e.severity == 'WARNING' || e.severity == 'ERROR') {
    console.log('mbgl:', e);
});
module.exports = {
    initRasterRender: function(renderConfig, awsConfig, vtconfig){
        var rasterRender = {
            renderers: [],
            awsS3Client: [],
            vtconfig: vtconfig
        const createPool = function(ratio, min, max, isEng, workerIndex) {
            const createRenderer = function(ratio, createCallback) {
                var renderer = new mbgl.Map({
                    ratio: ratio,
                    request: function(req, callback){
                        var opts = {
                            url: req.url,
                            encoding: null,
                            gzip: true
                        //"http://52.221.200.213:10101/v2/normal/{z}/{x}/{y}.pbf"
                        //console.log(req.url);
                        var parts = req.url.split('/');
                        var format = parts.slice(-1)[0].split('.')[1];
                        if(parts.length > 7 && format == 'pbf'){
                            var z = parts[5] | 0;
                            var x = parts[6] | 0;
                            var y = parts[7].split('.')[0] | 0;
                            var requestdata = {z: z, x: x, y: y, type:'pbf'}
                            var mqdata = {
                                channel: null,
                                msg: null,
                                requestdata: requestdata,
                                callback: function(error, data){
                                    var response = {};
                                    response.data = zlib.unzipSync(data);
                                    callback(null,  response);
                            renderVectorTile(rasterRender.vtconfig, mqdata);
                            //console.log('%d, %d, %d %s', z, x, y, format);
                        }else{
                            request(opts, function(error, res, body){
                                if(error){
                                    callback(error);
                                }else if(res.statusCode == 200 ){
                                    var response = {};
7172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
var isZip = false; //res.headers['content-encoding'] === 'gzip'; //console.log(res.headers['content-encoding']); if (res.headers.modified) { response.modified = new Date(res.headers.modified); } if (res.headers.expires) { response.expires = new Date(res.headers.expires); } if (res.headers.etag) { response.etag = res.headers.etag; } if(body){ //console.log(body); if(isZip){ response.data = zlib.unzipSync(body); }else{ response.data = body; } }else{ if(format == 'pbf'){ response.data = new Buffer(0); } } callback(null, response); }else{ var response = {}; if(format == 'pbf'){ response.data = new Buffer(0); } callback(null, response); //callback(new Error(JSON.parse(body).message)); } }); } } }); // renderer.load(require('./mapmagic_raster.json')); var style = require(renderConfig.style); var layers = style.layers; //console.log(layers.length); layers.map((layer,index) => { //console.log(layer.id); var layout = layer.layout; if(layout){ var textField = layout['text-field']; if(layout['icon-image']||textField){ if(layout['symbol-avoid-edges']==null){ layer.layout = Object.assign({},layout,{'symbol-avoid-edges': true}); //console.log(layer.layout); } } if(textField&&isEng){ var listTextField = textField.stops; if(listTextField){ var list = []; listTextField.map((field,index) => { if(field){ field[1] = field[1].replace('_th','_en'); } list.push(field); }); textField = Object.assign({},layout,{'stops': list }); layer.layout = Object.assign({},layout,{'text-field': textField }); }else{ layer.layout = Object.assign({},layout,{'text-field': textField.replace('_th','_en')}); } } } }); renderer.load(style); createCallback(null, renderer);
141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210
//create s3 rasterRender.awsS3Client[workerIndex] = new aws.S3({ accessKeyId: awsConfig.awsAccessKeyId, secretAccessKey: awsConfig.awsSecretAccessKey, region: awsConfig.awsRegion }); //"rtiles-mmg-2017-{name}-{lng}-{scale}" }; return new advancedPool.Pool({ min: min, max: max, create: createRenderer.bind(null, ratio), destroy: function(renderer) { renderer.release(); } }); }; rasterRender.renderers[1] = createPool(1, 4, 4, false, 1); rasterRender.renderers[2] = createPool(2, 4, 4, false, 2); rasterRender.renderers[3] = createPool(1, 4, 4, true, 3); rasterRender.renderers[4] = createPool(2, 4, 4, true, 4); return rasterRender; }, render: function(config, mqdata, rasterRender){ const getScale = function(scale) { return (scale || '@1x').slice(1, 2) | 0; }; const saveTileOnS3 = function(awsClient, awsParams, buffer){ awsParams['ContentType'] = 'image/png'; awsParams['Body'] = buffer; awsParams['ACL'] = 'private'; awsClient.putObject(awsParams, function(err, data){ if (err) { console.log(err, err.stack); } }); }; const sendResponseToClient = function(awsClient, channel, msg, buffer, awsParams){ if(awsParams){ saveTileOnS3(awsClient, awsParams, buffer) } channel.sendToQueue(msg.properties.replyTo, buffer, {correlationId: msg.properties.correlationId, type:'png'}); channel.ack(msg); }; const getTile = function(config, mqdata, rasterRender){ var requestdata = JSON.parse(mqdata.msg.content.toString()); var x = requestdata.x; var y = requestdata.y; var z = requestdata.z; var scale = getScale(requestdata.scale); var lng = requestdata.lng; var tileCenter = merc.ll([ ((x + 0.5) / (1 << z)) * (256 << z), ((y + 0.5) / (1 << z)) * (256 << z) ], z); console.log('get rt tile %d, %d, %d, %d', z, x, y, scale); var index = scale+(lng==='en'?2:0); var pool = rasterRender.renderers[index]; pool.acquire(function(err, renderer) { var awsClient = rasterRender.awsS3Client[index]; var key = z+'_'+x+'_'+y;
211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263
var hashKey = crc.crc32(key); var keyPath = hashKey+'/'+ z + '/' + x + '/' + y + '.png'; var awsParams = { Bucket: config.awsConfig.awsBucketNameFormat.replace('{name}', config.renderConfig.name) .replace('{lng}', lng) .replace('{scale}',(scale === 2)?'x2':'x1'), Key: keyPath }; awsClient.getObject(awsParams, function(err, awsdata) { if(err){ var params = { zoom: z-1, center: [tileCenter[0],tileCenter[1]] , bearing: 0, pitch: 0, width: tileSize, height: tileSize }; renderer.render(params, function(err, data) { pool.release(renderer); if (err){ console.log(err); return sendResponseToClient(awsClient, mqdata.channel, mqdata.msg, null , awsParams); } //console.log('Render complete!!'); var image = sharp(data, { raw: { width: tileSize*scale , height: tileSize*scale , channels: 4 } }); image.png({adaptiveFiltering: false}); image.toBuffer(function(err, buffer, info) { return sendResponseToClient(awsClient, mqdata.channel, mqdata.msg, buffer, awsParams); }); }); }else{ pool.release(renderer); console.log('raster tile have in s3'); return sendResponseToClient(awsClient, mqdata.channel, mqdata.msg, awsdata['Body'], null); } }); }); }; return getTile(config, mqdata, rasterRender); } };