Trying to retrieve url metadata in Express/NodeJS

I have a resource that contains a URL field entered by the user. I am trying to use this package: https://github.com/mozilla/page-metadata-parser to retrieve the title and description associated with the URL, and save this to the db on creation.

I have added the code modeled in the package documentation to the post request in Express and there are no errors, the new bookmark is created but the meta data values do not get returned.

Here is my model:

const mongoose = require('mongoose');

const { Schema } = mongoose;

const BookmarksSchema = new Schema({
  userId: {
    type: Schema.Types.ObjectId,
    required: true
  },
  url: {
    type: String,
    trim: true,
    required: true
  },

...

  title: {
    type: String,
    trim: true,
    required: false
  },
  description: {
    type: String,
    trim: true,
    required: false
  }
});

mongoose.model('Bookmarks', BookmarksSchema);

my create method:

const mongoose = require('mongoose');
const passport = require('passport');
const router = require('express').Router();
const auth = require('../auth');
const Bookmarks = mongoose.model('Bookmarks');

router.post('/', auth.required, (req, res, next) => {
  const userId = req.user.id;
  const bookmark = req.body.bookmark;

  if (!bookmark.url) {
    return res.status(422).json({
      errors: {
        url: 'is required',
      },
    });
  }

  const { getMetadata } = require('page-metadata-parser');
  const domino = require('domino');

  const url = bookmark.url;
  const response = fetch(url);
  const html = response.text();
  const doc = domino.createWindow(html).document;
  const metadata = getMetadata(doc, url);

  bookmark.userId = userId;
  bookmark.title = metadata.title;
  bookmark.description = metadata.description;

  const finalBookmark = new Bookmarks(bookmark);

  return finalBookmark.save()
    .then(() => res.json({ bookmark: finalBookmark }));
});

and package.json:

{
  "name": "server",
  "version": "1.0.0",
  "description": "",
  "main": "app.js",
  "dependencies": {
    "body-parser": "^1.18.3",
    "cors": "^2.8.5",
    "domino": "^2.1.3",
    "errorhandler": "^1.5.0",
    "express": "^4.16.4",
    "express-jwt": "^5.3.1",
    "express-session": "^1.15.6",
    "jsonwebtoken": "^8.5.1",
    "mongoose": "^5.4.20",
    "morgan": "^1.9.1",
    "page-metadata-parser": "^1.1.3",
    "passport": "^0.4.0",
    "passport-local": "^1.0.0",
    "path": "^0.12.7"
  },
  "devDependencies": {},
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1",
    "dev": "nodemon app"
  },
  "author": "",
  "license": "ISC"
}

Solution

Posting the answer here so we can mark the answer as correct.

The error arose due to the fact that the call to fetch() was an asynchronous call and the keyword await was not being used. The example on the NPM site found here:

https://www.npmjs.com/package/page-metadata-parser

Shows that they used await on the fetch() call. In order to use await the anonymous callback function, the function that starts with (req, res, next) has to have the keyword async in front of it. The call should then look like this:

router.post('/', auth.required, async (req, res, next) => {
     // Do your stuff here as before.
     const url = bookmark.url;
     const response = await fetch(url);
     const html = response.text();
     const doc = domino.createWindow(html).document;
     const metadata = getMetadata(doc, url);
     // Finish stuff here.
});

Now response is being populated and the program waits until the fetch call is complete before moving forward, thus filling out the remaining variables and being able to grab the meta data.