sgdpa/public/bundles/apiplatform/fetch/fetch.js

'use strict';

var http = require('http');
var https = require('https');
var urllib = require('url');
var utillib = require('util');
var zlib = require('zlib');
var dns = require('dns');
var Stream = require('stream').Readable;
var CookieJar = require('./cookiejar').CookieJar;
var encodinglib = require('encoding');
var net = require('net');

var USE_ALLOC = typeof Buffer.alloc === 'function';

exports.FetchStream = FetchStream;
exports.CookieJar = CookieJar;
exports.fetchUrl = fetchUrl;

function FetchStream(url, options) {
    Stream.call(this);

    options = options || {};

    this.url = url;
    if (!this.url) {
        return this.emit('error', new Error('url not defined'));
    }

    this.userAgent = options.userAgent || 'FetchStream';

    this._redirect_count = 0;

    this.options = options || {};
    this.normalizeOptions();

    // prevent errors before 'error' handler is set by defferring actions
    if (typeof setImmediate !== 'undefined') {
        setImmediate(this.runStream.bind(this, url));
    } else {
        process.nextTick(this.runStream.bind(this, url));
    }
    this.responseBuffer = USE_ALLOC ? Buffer.alloc(0, '', 'binary') : new Buffer(0, 'binary');
    this.ended = false;
    this.readyToRead = 0;
}
utillib.inherits(FetchStream, Stream);

FetchStream.prototype._read = function (size) {
    if (this.ended && this.responseBuffer.length === 0) {
        this.push(null);
        return;
    }
    this.readyToRead += size;
    this.drainBuffer();
};

FetchStream.prototype.drainBuffer = function () {
    if (this.readyToRead === 0) {
        return;
    }
    if (this.responseBuffer.length === 0) {
        return;
    }
    var push;
    var rest;
    var restSize;

    if (this.responseBuffer.length > this.readyToRead) {
        push = USE_ALLOC ? Buffer.alloc(this.readyToRead, '', 'binary') : new Buffer(this.readyToRead, 'binary');
        this.responseBuffer.copy(push, 0, 0, this.readyToRead);
        restSize = this.responseBuffer.length - this.readyToRead;
        rest = USE_ALLOC ? Buffer.alloc(restSize, '', 'binary') : new Buffer(restSize, 'binary');
        this.responseBuffer.copy(rest, 0, this.readyToRead);
    } else {
        push = this.responseBuffer;
        rest = USE_ALLOC ? Buffer.alloc(0, '', 'binary') : new Buffer(0, 'binary');
    }
    this.responseBuffer = rest;
    this.readyToRead = 0;
    if (this.options.encoding) {
        this.push(push, this.options.encoding);
    } else {
        this.push(push);
    }
};

FetchStream.prototype.destroy = function (ex) {
    this.emit('destroy', ex);
};

FetchStream.prototype.normalizeOptions = function () {

    // cookiejar
    this.cookieJar = this.options.cookieJar || new CookieJar();

    // default redirects - 10
    // if disableRedirect is set, then 0
    if (!this.options.disableRedirect && typeof this.options.maxRedirects !== 'number' &&
        !(this.options.maxRedirects instanceof Number)) {
        this.options.maxRedirects = 10;
    } else if (this.options.disableRedirects) {
        this.options.maxRedirects = 0;
    }

    // normalize header keys
    // HTTP and HTTPS takes in key names in case insensitive but to find
    // an exact value from an object key name needs to be case sensitive
    // so we're just lowercasing all input keys
    this.options.headers = this.options.headers || {};

    var keys = Object.keys(this.options.headers);
    var newheaders = {};
    var i;

    for (i = keys.length - 1; i >= 0; i--) {
        newheaders[keys[i].toLowerCase().trim()] = this.options.headers[keys[i]];
    }

    this.options.headers = newheaders;

    if (!this.options.headers['user-agent']) {
        this.options.headers['user-agent'] = this.userAgent;
    }

    if (!this.options.headers.pragma) {
        this.options.headers.pragma = 'no-cache';
    }

    if (!this.options.headers['cache-control']) {
        this.options.headers['cache-control'] = 'no-cache';
    }

    if (!this.options.disableGzip) {
        this.options.headers['accept-encoding'] = 'gzip, deflate';
    } else {
        delete this.options.headers['accept-encoding'];
    }

    // max length for the response,
    // if not set, default is Infinity
    if (!this.options.maxResponseLength) {
        this.options.maxResponseLength = Infinity;
    }

    // method:
    // defaults to GET, or when payload present to POST
    if (!this.options.method) {
        this.options.method = this.options.payload || this.options.payloadSize ? 'POST' : 'GET';
    }

    // set cookies
    // takes full cookie definition strings as params
    if (this.options.cookies) {
        for (i = 0; i < this.options.cookies.length; i++) {
            this.cookieJar.setCookie(this.options.cookies[i], this.url);
        }
    }

    // rejectUnauthorized
    if (typeof this.options.rejectUnauthorized === 'undefined') {
        this.options.rejectUnauthorized = true;
    }
};

FetchStream.prototype.parseUrl = function (url) {
    var urlparts = urllib.parse(url, false, true),
        transport,
        urloptions = {
            host: urlparts.hostname || urlparts.host,
            port: urlparts.port,
            path: urlparts.pathname + (urlparts.search || '') || '/',
            method: this.options.method,
            rejectUnauthorized: this.options.rejectUnauthorized
        };

    switch (urlparts.protocol) {
        case 'https:':
            transport = https;
            break;
        case 'http:':
        default:
            transport = http;
            break;
    }

    if (transport === https) {
        if('agentHttps' in this.options){
            urloptions.agent = this.options.agentHttps;
        }
        if('agent' in this.options){
            urloptions.agent = this.options.agent;
        }
    } else {
        if('agentHttp' in this.options){
            urloptions.agent = this.options.agentHttp;
        }
        if('agent' in this.options){
            urloptions.agent = this.options.agent;
        }
    }

    if (!urloptions.port) {
        switch (urlparts.protocol) {
            case 'https:':
                urloptions.port = 443;
                break;
            case 'http:':
            default:
                urloptions.port = 80;
                break;
        }
    }

    urloptions.headers = this.options.headers || {};

    if (urlparts.auth) {
        var buf = USE_ALLOC ? Buffer.alloc(Buffer.byteLength(urlparts.auth), urlparts.auth) : new Buffer(urlparts.auth);
        urloptions.headers.Authorization = 'Basic ' + buf.toString('base64');
    }

    return {
        urloptions: urloptions,
        transport: transport
    };
};

FetchStream.prototype.setEncoding = function (encoding) {
    this.options.encoding = encoding;
};

FetchStream.prototype.runStream = function (url) {
    var url_data = this.parseUrl(url),
        cookies = this.cookieJar.getCookies(url);

    if (cookies) {
        url_data.urloptions.headers.cookie = cookies;
    } else {
        delete url_data.urloptions.headers.cookie;
    }

    if (this.options.payload) {
        url_data.urloptions.headers['content-length'] = Buffer.byteLength(this.options.payload || '', 'utf-8');
    }

    if (this.options.payloadSize) {
        url_data.urloptions.headers['content-length'] = this.options.payloadSize;
    }

    if (this.options.asyncDnsLoookup) {
        var dnsCallback = (function (err, addresses) {
            if (err) {
                this.emit('error', err);
                return;
            }

            url_data.urloptions.headers.host = url_data.urloptions.hostname || url_data.urloptions.host;
            url_data.urloptions.hostname = addresses[0];
            url_data.urloptions.host = url_data.urloptions.headers.host + (url_data.urloptions.port ? ':' + url_data.urloptions.port : '');

            this._runStream(url_data, url);
        }).bind(this);

        if (net.isIP(url_data.urloptions.host)) {
            dnsCallback(null, [url_data.urloptions.host]);
        } else {
            dns.resolve4(url_data.urloptions.host, dnsCallback);
        }
    } else {
        this._runStream(url_data, url);
    }
};

FetchStream.prototype._runStream = function (url_data, url) {

    var req = url_data.transport.request(url_data.urloptions, (function (res) {

        // catch new cookies before potential redirect
        if (Array.isArray(res.headers['set-cookie'])) {
            for (var i = 0; i < res.headers['set-cookie'].length; i++) {
                this.cookieJar.setCookie(res.headers['set-cookie'][i], url);
            }
        }

        if ([301, 302, 303, 307, 308].indexOf(res.statusCode) >= 0) {
            if (!this.options.disableRedirects && this.options.maxRedirects > this._redirect_count && res.headers.location) {
                this._redirect_count++;
                req.destroy();
                this.runStream(urllib.resolve(url, res.headers.location));
                return;
            }
        }

        this.meta = {
            status: res.statusCode,
            responseHeaders: res.headers,
            finalUrl: url,
            redirectCount: this._redirect_count,
            cookieJar: this.cookieJar
        };

        var curlen = 0,
            maxlen,

            receive = (function (chunk) {
                if (curlen + chunk.length > this.options.maxResponseLength) {
                    maxlen = this.options.maxResponseLength - curlen;
                } else {
                    maxlen = chunk.length;
                }

                if (maxlen <= 0) {
                    return;
                }

                curlen += Math.min(maxlen, chunk.length);
                if (maxlen >= chunk.length) {
                    if (this.responseBuffer.length === 0) {
                        this.responseBuffer = chunk;
                    } else {
                        this.responseBuffer = Buffer.concat([this.responseBuffer, chunk]);
                    }
                } else {
                    this.responseBuffer = Buffer.concat([this.responseBuffer, chunk], this.responseBuffer.length + maxlen);
                }
                this.drainBuffer();
            }).bind(this),

            error = (function (e) {
                this.ended = true;
                this.emit('error', e);
                this.drainBuffer();
            }).bind(this),

            end = (function () {
                this.ended = true;
                if (this.responseBuffer.length === 0) {
                    this.push(null);
                }
            }).bind(this),

            unpack = (function (type, res) {
                var z = zlib['create' + type]();
                z.on('data', receive);
                z.on('error', error);
                z.on('end', end);
                res.pipe(z);
            }).bind(this);

        this.emit('meta', this.meta);

        if (res.headers['content-encoding']) {
            switch (res.headers['content-encoding'].toLowerCase().trim()) {
                case 'gzip':
                    return unpack('Gunzip', res);
                case 'deflate':
                    return unpack('InflateRaw', res);
            }
        }

        res.on('data', receive);
        res.on('end', end);

    }).bind(this));

    req.on('error', (function (e) {
        this.emit('error', e);
    }).bind(this));

    if (this.options.timeout) {
        req.setTimeout(this.options.timeout, req.abort.bind(req));
    }
    this.on('destroy', req.abort.bind(req));

    if (this.options.payload) {
        req.end(this.options.payload);
    } else if (this.options.payloadStream) {
        this.options.payloadStream.pipe(req);
        this.options.payloadStream.resume();
    } else {
        req.end();
    }
};

function fetchUrl(url, options, callback) {
    if (!callback && typeof options === 'function') {
        callback = options;
        options = undefined;
    }
    options = options || {};

    var fetchstream = new FetchStream(url, options),
        response_data, chunks = [],
        length = 0,
        curpos = 0,
        buffer,
        content_type,
        callbackFired = false;

    fetchstream.on('meta', function (meta) {
        response_data = meta;
        content_type = _parseContentType(meta.responseHeaders['content-type']);
    });

    fetchstream.on('data', function (chunk) {
        if (chunk) {
            chunks.push(chunk);
            length += chunk.length;
        }
    });

    fetchstream.on('error', function (error) {
        if (error && error.code === 'HPE_INVALID_CONSTANT') {
            // skip invalid formatting errors
            return;
        }
        if (callbackFired) {
            return;
        }
        callbackFired = true;
        callback(error);
    });

    fetchstream.on('end', function () {
        if (callbackFired) {
            return;
        }
        callbackFired = true;

        buffer = USE_ALLOC ? Buffer.alloc(length) : new Buffer(length);
        for (var i = 0, len = chunks.length; i < len; i++) {
            chunks[i].copy(buffer, curpos);
            curpos += chunks[i].length;
        }

        if (content_type.mimeType === 'text/html') {
            content_type.charset = _findHTMLCharset(buffer) || content_type.charset;
        }

        content_type.charset = (options.overrideCharset || content_type.charset || 'utf-8').trim().toLowerCase();


        if (!options.disableDecoding && !content_type.charset.match(/^utf-?8$/i)) {
            buffer = encodinglib.convert(buffer, 'UTF-8', content_type.charset);
        }

        if (options.outputEncoding) {
            return callback(null, response_data, buffer.toString(options.outputEncoding));
        } else {
            return callback(null, response_data, buffer);
        }

    });
}

function _parseContentType(str) {
    if (!str) {
        return {};
    }
    var parts = str.split(';'),
        mimeType = parts.shift(),
        charset, chparts;

    for (var i = 0, len = parts.length; i < len; i++) {
        chparts = parts[i].split('=');
        if (chparts.length > 1) {
            if (chparts[0].trim().toLowerCase() === 'charset') {
                charset = chparts[1];
            }
        }
    }

    return {
        mimeType: (mimeType || '').trim().toLowerCase(),
        charset: (charset || 'UTF-8').trim().toLowerCase() // defaults to UTF-8
    };
}

function _findHTMLCharset(htmlbuffer) {

    var body = htmlbuffer.toString('ascii'),
        input, meta, charset;

    if ((meta = body.match(/<meta\s+http-equiv=["']content-type["'][^>]*?>/i))) {
        input = meta[0];
    }

    if (input) {
        charset = input.match(/charset\s?=\s?([a-zA-Z\-0-9]*);?/);
        if (charset) {
            charset = (charset[1] || '').trim().toLowerCase();
        }
    }

    if (!charset && (meta = body.match(/<meta\s+charset=["'](.*?)["']/i))) {
        charset = (meta[1] || '').trim().toLowerCase();
    }

    return charset;
}