index.js

const pkgJson = require("./package.json");
const ShelfPack = require('@mapbox/shelf-pack');
const sharp = require('sharp');
const genEtag = require('etag');
const fresh = require('fresh');
const pLimit = require('p-limit');
const fetch = require('node-fetch');
const fs = require("fs");
const debug = require("debug")(pkgJson.name);


// Generated by sharp (./scripts/gen_empty_img.sh)
const EMPTY_PNG = fs.readFileSync(__dirname+"/empty_image.png");
  

function qsToggle (qs, k) {
  const valid = ["", "true", "1"]
  return valid.includes(qs[k]);
}

function json (imgs, opts={}) {
  opts = {
    pixelRatio: 1,
    ...opts,
  };
  const {pixelRatio} = opts;
  const sprite = new ShelfPack(1, 1, { autoResize: true });

  const clampedSizedImages = imgs.map(img => {
    return {
      ...img,
      width: Math.round(img.width),
      height: Math.round(img.height),
    }
  })
  const results = sprite.pack(clampedSizedImages, { inPlace: true });

  const out = {};
  results.forEach(item => {
    out[item.id] = {
      "pixelRatio": pixelRatio,
      "width": item.w*pixelRatio,
      "height": item.h*pixelRatio,
      "x": item.x*pixelRatio,
      "y": item.y*pixelRatio,
    };
  })
  return {
    width: sprite.w*pixelRatio,
    height: sprite.h*pixelRatio,
    images: clampedSizedImages,
    boxes: out,
    pixelRatio,
  };
}

async function png (spriteJson, opts={}) {
  const {pixelRatio, boxes, images, width, height} = spriteJson;
  const {imageConcurrency} = {
    pngCompression: 7,
    imageConcurrency: 100,
    ...opts,
  };

  function insertImageUrl (def) {
    const img = boxes[def.id];
    return {
      ...def,
      ...img,
    };
  }

  function blankCanvas () {
    return sharp({
      create: {
        width,
        height,
        channels: 4,
        background: { r: 0, g: 0, b: 0, alpha: 0 }
      }
    });
  }

  const cache = {
    outputImage: await blankCanvas().png({compressionLevel: 0}).toBuffer(),
    workingPromise: null
  };

  const queue = [];

  // Basically a really ugly serialized function. So we can add the image to
  // the output buffer as we go
  const composite = async (imageObj, flush=false) => {
    // Wait for other promises to succeed.
    while (cache.workingPromise) {
      await cache.workingPromise;
    }

    if (imageObj) {
      queue.push({
        input: imageObj.input,
        top: imageObj.y,
        left: imageObj.x,
      });
    }

    if (flush || queue.length >= imageConcurrency) {
      cache.workingPromise = blankCanvas().composite([
        {
          input: cache.outputImage,
          top: 0,
          left: 0,
        },
        ...queue,
      ]).png({compressionLevel: flush ? opts.pngCompression : 0}).toBuffer();

      cache.outputImage = await cache.workingPromise;
      queue.splice(0);
      cache.workingPromise = null;
    }
  };

  const imgs = await fetchImages(
    composite,
    images.map(insertImageUrl),
    opts,
  );

  // Flush...
  await composite(null, true);

  const missingImages = imgs.filter(img => img.missing).map(img => img.id);

  if (!imgs.length) {
    return EMPTY_PNG;
  }

  return {
    missingImages,
    buffer: cache.outputImage,
  };
}

async function fetchImage (composite, imgDef) {
  try {
    const resp = await imgDef.resp;
    const {status} = resp;
    if (status < 200 || status >= 300) {
      debug("Not found: '%s'", imgDef.url);

      await composite({
        ...imgDef,
        width: 1,
        height: 1,
        input: EMPTY_PNG,
      })
      return {
        ...imgDef,
        missing: true,
      }
    }
    if (imgDef.buffer) {
      await composite({
        ...imgDef,
        input: imgDef.buffer
      });
      const out = {...imgDef};
      delete out["buffer"];
      return out;
    }
    else if (imgDef.url.match(/\.svg$/i)) {
      const inBuffer = await resp.buffer();

      const image = sharp(inBuffer);
      const metadata = await image.metadata();
      let density = metadata.density;
      if (imgDef.width && metadata.width && imgDef.width > metadata.width) {
        // We must limit to 2400
        // <https://github.com/lovell/sharp/issues/1421#issuecomment-514446234>
        density = Math.min(
          2400,
          (72 * imgDef.width) / metadata.width
        );
      }

      const buffer = await sharp(inBuffer, {density})
      .resize(imgDef.width, imgDef.height)
      .png({compressionLevel: 0})
      .toBuffer();

      await composite({
        ...imgDef,
        input: buffer,
      });

      return {
        ...imgDef,
      };
    }
    else {
      const inBuffer = await resp.buffer();

      const buffer = await sharp(inBuffer)
        .resize(imgDef.width, imgDef.height)
        .png({compressionLevel: 0})
        .toBuffer();

      await composite({
        ...imgDef,
        input: buffer,
      })

      return {
        ...imgDef,
      };
    }
  }
  catch (_err) {
    console.warn(_err);
    await composite({
      ...imgDef,
      width: 1,
      height: 1,
      input: EMPTY_PNG,
    });
    return {
      ...imgDef,
      missing: true,
    };
  }
}

async function fetchImages (composite, imgs, opts={}) {
  opts = {
    processConcurrency: 100,
    concurrency: 100,
    ...opts,
  };
  const requestLimit = pLimit(opts.concurrency);
  const processLimit = pLimit(opts.processConcurrency);

  await Promise.all(imgs.map(img => {
    return requestLimit(() => {
      img.resp = fetch(img.url)
    });
  }));

  const promises = imgs.map(img => {
    return processLimit(() => fetchImage(composite, img))
  });

  return Promise.all(promises);
}

async function convert(imgs) {
  const outJson = json(imgs);
  const {buffer, missingImages} = await png(outJson);
  return {
    json: outJson.boxes,
    missingImages,
    buffer,
  };
}

function middleware (resolver, opts={}) {
    const {
      concurrency,
      missingImageRetryInterval,
    } = {
      concurrency: 100,
      processConcurrency: 100,
      missingImageRetryInterval: 60,
      ...opts
    };

    return async function (req, res, next) {
    try {
      const imgs = await resolver(req);
      if (!imgs) {
        res.status(404).end();
        return;
      }

      const urlMatches = req.originalUrl.match(/(?:@([0-9]+)x)?\.(png|json)$/);
      if (!urlMatches) {
        next(new Error("Expected URL to have suffix of format /(@[0.9]+x)?\.(png|json)/"))
        return;
      }

      const debugging = qsToggle(req.query, "debug");
      const pixelRatio = parseInt(urlMatches[1], 10) || 1;
      const format = urlMatches[2];

      const spriteJson = json(imgs, {pixelRatio});
      const apiResp = JSON.stringify(spriteJson.boxes, null, debugging ? 2 : 0);

      const etag = genEtag(apiResp);
      res.setHeader("etag", etag);

      // Etag check because it's cheap here and we don't have to do any JSON processing.
      if (fresh(req.headers, {etag})) {
        // Just use the browser/cdn cache.
        res.status(304).end();
        return;
      }

      if (format === "json") {
        res.setHeader("content-type", "text/json");
        res.send(apiResp);
        return;
      }
      else if (format === "png") {
        const start = Date.now();
        const {buffer, missingImages} = await png(spriteJson, opts);
        const numberOfImages = spriteJson.images.length;
        const took = Date.now() - start;
        const perImageTook = (took/numberOfImages).toFixed(4);
        debug(`png generation took: ${took}ms, ${numberOfImages} images, ${perImageTook}ms per image`);
        if (missingImages.length > 0) {
          const etag = genEtag(buffer);
          res.setHeader("etag", etag);
          res.setHeader("Cache-Control", `public, max-age=${missingImageRetryInterval}`)
        }
        res.setHeader("content-type", "image/png");
        res.send(buffer).end();
      }
      else {
        throw new Error("Unexpected error");
      }
    }
    catch(err) {
      next(err);
      return;
    }
  }
}

module.exports = {
  middleware,
  png,
  json,
};