Gatsby build slow (800 pages)

[gafasstudio]

Description

We build a website with gatsby (npm run build), the data comes from Contentful and it is build on Netlify. The website has around 800 pages. But it takes 15 minutes to build the site on Netlify and Netlify has a 15min limit. Locally it also takes 10 minutes to build the site, which seems a bit long or is it normal?

Deploy log can be found in the pdf
deploy_log_netlify.txt.pdf

Environment

Gatsby version: 1.9.149
Node.js version: v8.9.4
Operating System: OSX, Linux (Netlify)

File contents (if changed):

gatsby-config.js:

module.exports = {
  siteMetadata: {
    title: `website`,
  },
  plugins: [
    {
      resolve: `gatsby-source-contentful`,
      options: {
        spaceId: `xxxxx`,
[deploy_log_netlify.txt.pdf](https://github.com/gatsbyjs/gatsby/files/1681794/deploy_log_netlify.txt.pdf)



        accessToken: `xxxxx`,
      },
    },
    {
        resolve: `gatsby-source-contentful`,
        options: {
            spaceId: `xxxxx`,
            accessToken: `xxxxxxx`,
        },
    },
    {
      resolve: `gatsby-plugin-nprogress`,
      options: {
        // Setting a color is optional.
        color: `#C2A572`,
        // Disable the loading spinner.
        showSpinner: false,
      },
    },
    {
      resolve: `gatsby-plugin-google-analytics`,
      options: {
        trackingId: "XX-XXXX-XX",
      },
    },
    `gatsby-transformer-remark`,
  ],

};

package.json:

{
  "name": "website",
  "version": "1.0.0",
  "license": "MIT",
  "main": "n/a",
  "dependencies": {
    "@google/maps": "^0.4.5",
    "babel-polyfill": "^6.26.0",
    "blazy": "^1.8.2",
    "bowser": "^1.8.1",
    "classnames": "^2.2.5",
    "contentful-management": "^4.1.1",
    "flickity": "^2.0.10",
    "flickity-bg-lazyload": "^1.0.0",
    "gatsby": "^1.9.77",
    "gatsby-link": "^1.6.20",
    "gatsby-plugin-google-analytics": "^1.0.13",
    "gatsby-plugin-nprogress": "^1.0.8",
    "gatsby-plugin-offline": "^1.0.10",
    "gatsby-source-contentful": "^1.3.18",
    "gatsby-transformer-remark": "^1.7.21",
    "gsap": "^1.20.3",
    "hash-object": "^0.1.7",
    "ie-array-find-polyfill": "^1.1.0",
    "in-view": "^0.6.1",
    "ismobilejs": "^0.4.1",
    "lodash": "^4.16.4",
    "mandrill-api": "^1.0.45",
    "moment": "^2.19.1",
    "netlify-cli": "^1.2.2",
    "progressbar.js": "^1.0.1",
    "prop-types": "^15.5.10",
    "react-click-outside": "^2.3.1",
    "react-document-meta": "^2.1.2",
    "react-flatpickr": "^3.6.0",
    "react-flickity-component": "^1.1.0",
    "react-google-maps": "^9.1.0",
    "react-headrooms": "^1.0.6",
    "react-inview": "^0.0.6",
    "react-inview-js": "^1.2.7",
    "react-markdown": "^2.5.0",
    "react-masonry-component": "^6.0.1",
    "react-places-autocomplete": "^5.4.3",
    "react-player": "^0.25.2",
    "react-progressbar.js": "^0.2.0",
    "react-scrollable-anchor": "^0.5.0",
    "react-visibility-sensor": "^3.11.0",
    "save": "^2.3.1",
    "scrollprogress": "^3.0.2",
    "slash": "^1.0.0",
    "wheel-react": "^1.0.3"
  },
  "scripts": {
    "develop": "npm-run-all --parallel develop:*",
    "develop:gatsby": "gatsby develop",
    "develop:css": "postcss src/css/app.css -o public/app.css -w",
    "develop:css-static": "postcss src/css/app.css -o static/app.css -w",
    "build": "npm-run-all build:gatsby",
    "build:gatsby": "node --max-old-space-size=8192 node_modules/.bin/gatsby build",
    "build:css": "postcss src/css/app.css -o static/app.css",
    "start": "gatsby serve",
    "precommit": "lint-staged",
    "prettify": "prettier --trailing-comma es5 --single-quote --print-width 100 --write \"src/**/*.js\"",
    "prettify:g": "prettier --trailing-comma es5 --single-quote --print-width 100 --write \"gatsby-node.js\"",
    "fix": "eslint --fix src",
    "lint": "eslint src"
  },
  "devDependencies": {
    "babel-eslint": "7.2.3",
    "eslint": "3.19.0",
    "eslint-config-react-app": "^2.0.0",
    "eslint-plugin-flowtype": "2.33.0",
    "eslint-plugin-import": "2.2.0",
    "eslint-plugin-jsx-a11y": "5.0.1",
    "eslint-plugin-react": "7.0.1",
    "gatsby-cli": "^1.1.11",
    "husky": "^0.14.3",
    "jspolyfill-array.prototype.find": "^0.1.3",
    "lint-staged": "^4.1.0",
    "npm-run-all": "^4.1.2",
    "postcss": "^6.0.14",
    "postcss-cli": "^4.1.1",
    "postcss-cssnext": "^3.0.2",
    "postcss-import": "^11.0.0",
    "prettier": "^1.6.1"
  },
  "lint-staged": {
    "src/**/*.js": [
      "prettier --trailing-comma es5 --single-quote --print-width 100 --write",
      "git add"
    ]
  },
  "eslintConfig": {
    "extends": [
      "react-app"
    ],
    "globals": {
      "graphql": false,
      "$": false,
      "google": false,
      "history": false
    }
  }
}

`gatsby-node.js`: const _ = require('lodash');
const Promise = require('bluebird');
const path = require('path');
const slash = require(`slash`);
const storeLocator = require('./node/storeLocator');
const queries = require('./node/queries');

//
function createSinglePages(key, createPage, result, language, slug, conditional = 'id') {
  const template = path.resolve(`./src/templates/${key}.js`);
  _.each(_.filter(result, edge => edge.node.node_locale == language), edge => {
    if (edge.node[conditional]) {
      createPage({
        path: `${language}/${_.replace(slug, ':slug', edge.node.slug)}`,
        component: slash(template),
        context: {
          locale: language,
          id: edge.node.id,
          navId: `c1DzxdxolMIwGMs2wMucG8W${language === 'nl-BE' ? '' : `___${language}`}`,
        },
      });
    }
  });
}

function createWikiPages(key, category, result, slug, createPage, language) {
  var array = _.filter(result, edge => edge.node.type == category);
  array = _.filter(array, edge => edge.node.node_locale == language);
  
  const template = path.resolve(`./src/templates/${key}.js`);
  _.each(array, edge => {
    _.each(edge.node.wikiSubcategorien, category => {
      if (category.id) {
        createPage({
          path: `${language}/${_.replace(slug, ':slug', category.slug)}`,
          component: slash(template),
          context: {
            locale: language,
            id: category.id,
            navId: `c1DzxdxolMIwGMs2wMucG8W${language === 'nl-BE' ? '' : `___${language}`}`,
          },
        });
        _.each(category.wiki, wiki => {
          if (wiki.id) {
            createPage({
              path: `${language}/${_.replace(slug, ':slug', category.slug)}/${wiki.slug}`,
              component: slash(template),
              context: {
                locale: language,
                id: wiki.id,
                navId: `c1DzxdxolMIwGMs2wMucG8W${language === 'nl-BE' ? '' : `___${language}`}`,
              },
            });
          }
        });
      }
    });
  });
}

function AllPagesCreated(totalPages, resolveFn, ttl = 6000000) {
  this.totalPages = totalPages;
  this.pagesChecked = 0;

  const id = setTimeout(() => {
    console.log(
      '\n=====> TIMEOUT REACHED, STOPPING PAGE GENERATION! Only ' +
        this.pagesChecked +
        ' of ' +
        this.totalPages +
        ' expected pages generated! <===='
    );
    resolveFn();
  }, ttl);

  this.check = function check(key = 'default') {
    this.pagesChecked++;
    console.log(this.pagesChecked, this.totalPages, key);
    if (this.pagesChecked >= this.totalPages) {
      clearTimeout(id);
      resolveFn();
    }
  };
}

function createPages(graphql, createPage, resolve, reject) {
  graphql(`
    {
      allContentfulTemplatePage {
        edges {
          node {
              disclaimer {
              slug
              node_locale
            }
            horeca {
              slug
              node_locale
            }
            error404 {
              slug
              node_locale
            }
            contact {
              slug
              node_locale
            }
            brandCategories {
              slug
              node_locale
            }
            breweryVisit {
              slug
              node_locale
            }
            jobsSingle {
              slug
              node_locale
            }
            jobs {
              slug
              node_locale
            }
            brandCategorySingle {
              slug
              node_locale
            }
            history {
              slug
              node_locale
            }
            brewery {
              slug
              node_locale
            }
            news {
              slug
              node_locale
            }
            brandSingle {
              slug
              node_locale
            }
            singleStory {
              slug
              node_locale
            }
            sitemap {
              slug
              node_locale
            }
            sellingPoints {
              slug
              node_locale
            }
            immo {
              slug
              node_locale
            }
            immoSingle {
              slug
              node_locale
            }
            newsSingle {
              slug
              node_locale
            }
            eventSingle {
              slug
              node_locale
            }
            logos {
              slug
              node_locale
            }
            horecaWiki {
              slug
              node_locale
            }
            bedrijfWiki {
              slug
              node_locale
            }
            breweryProcess {
              slug
              node_locale
            }
            singlePress {
              slug
              node_locale
            }
          }
        }
      }
    }
  `).then(result => {
    if (result.errors) {
      reject(result.errors);
    }
    const numPages = _.reduce(
      result.data.allContentfulTemplatePage.edges,
      (total, languages, lang) => {
        return (
          total +
          _.reduce(
            languages.node,
            (total, pages, pageName) => {
              if (pages && !_.isArray(pages)) {
                console.warn(`Template ${pageName} should be a reference to multiple Slugs!`);
                return total;
              }
              return pages ? total + pages.length : total;
            },
            0
          )
        );
      },
      0
    );

const allPagesCreated = new AllPagesCreated(numPages, resolve);

    _.each(result.data.allContentfulTemplatePage.edges, (edge) => {
      const obj = _.keys(edge.node);
      const firstKey = obj[0];

      const homeTemplate = path.resolve(`./src/templates/home.js`);

      if (edge.node[firstKey]) {
        const language = edge.node[firstKey][0].node_locale;
        console.log(`----> ${language} <----`);
        createPage({
          path: `${language}`,
          component: slash(homeTemplate),
          context: {
            locale: language,
            navId: `c1DzxdxolMIwGMs2wMucG8W${language === 'nl-BE' ? '' : `___${language}`}`,
          },
        });
      }

      _.forEach(edge.node, (value, key) => {
        if (edge.node[key]) {
          const language = edge.node[key][0].node_locale;
          const slug = edge.node[key][0].slug;

          if (key === 'error404') {
            const errorTemplate = path.resolve(`./src/templates/404.js`);
            createPage({
              path: '/404/',
              component: slash(errorTemplate),
              context: {
                locale: 'nl-BE',
                navId: `c1DzxdxolMIwGMs2wMucG8W${language === 'nl-BE' ? '' : `___${language}`}`,
              },
            });
            allPagesCreated.check('404');
            return;
          }

          if (key === 'newsSingle') {
            queries.singleNews(graphql).then(result => {
              if (result.errors) {
                reject(result.errors);
              }

              createSinglePages(
                key,
                createPage,
                result.data.allContentfulNews.edges,
                language,
                slug
              );
              allPagesCreated.check('nieuwsSingle');
            });
            return;
          }
          
          if (key === 'horecaWiki' || key === 'bedrijfWiki') {
            queries.wiki(graphql).then(result => {
              if (result.errors) {
                reject(result.errors);
              }
              const category = key === 'horecaWiki' ? 'Horeca' : 'Bedrijf';
              createWikiPages(
                key,
                category,
                result.data.allContentfulWikiCategory.edges,
                slug,
                createPage,
                language
              );
              allPagesCreated.check('wikis');
            });
            return;
          }
          if (key === 'singlePress'){
            queries.singlePress(graphql).then(result => {
              if (result.errors) {
                reject(result.errors);
              }
              createSinglePages(key, createPage, result.data.allContentfulPress.edges, language, slug, 'slug');
              allPagesCreated.check('singlePress');
            });

            return;
          }
          if (key === 'immoSingle') {
            queries.singleImmo(graphql).then(result => {
              if (result.errors) {
                reject(result.errors);
              }
              createSinglePages(
                key,
                createPage,
                result.data.allContentfulProperty.edges,
                language,
                slug
              );
              allPagesCreated.check('immoSingle');
            });
            return;
          }
          if(key === 'sellingPoints'){
            const template = path.resolve(`./src/templates/${key}.js`);

            storeLocator.getLatLng().then(sellingPoints => {
              queries.singleAbbreviation(graphql)
                .then(result => {
                  if (result.errors){
                    reject(result.errors)
                  }
                  createPage({
                    path: `${language}/${slug}`,
                    component: slash(template),
                    context: {
                      locale: language,
                      navId: `c1DzxdxolMIwGMs2wMucG8W${language === 'nl-BE' ? '' : `___${language}`}`,
                      sellingPoints,
                    },
                  });
                  _.each(result.data.allContentfulAbbreviations.edges, edge => {
                    if(edge.node.slug){
                      createPage({
                        path: `${language}/${slug}/${edge.node.slug}`,
                        component: slash(template),
                        context: {
                          locale: language,
                          id: edge.node.id,
                          navId: `c1DzxdxolMIwGMs2wMucG8W${language === 'nl-BE' ? '' : `___${language}`}`,
                          sellingPoints,
                        },
                      });
                    }
                  });
                  allPagesCreated.check('sellingPoints');  });
            });
            return;
          }
          if (key === 'logos') {
            queries.logos(graphql).then(result => {
              if (result.errors) {
                reject(result.errors);
              }
              _.each(edge.node[key], item => {
                createSinglePages(
                  key,
                  createPage,
                  result.data.allContentfulPackshotCategory.edges,
                  language,
                  item.slug
                );
                allPagesCreated.check('logos');
              });
            });
            return;
          }
          if (key === 'singleStory') {
            queries.singleStory(graphql).then(result => {
              if (result.errors) {
                reject(result.errors);
              }
              createSinglePages(
                key,
                createPage,
                result.data.allContentfulVerhaal.edges,
                language,
                slug
              );
              allPagesCreated.check('singleStory');
            });
            return;
          }
          if (key === 'jobsSingle') {
            queries.singleJob(graphql).then(result => {
              if (result.errors) {
                reject(result.errors);
              }
              createSinglePages(
                key,
                createPage,
                result.data.allContentfulJob.edges,
                language,
                slug,
                "id"
              );
            });
            queries.extraJobs(graphql).then(result => {
              if (result.errors) {
                reject(result.errors);
              }
              createSinglePages(
                key,
                createPage,
                result.data.allContentfulJobsOther.edges,
                language,
                slug
              );
            });
            allPagesCreated.check('JobsSingle');
            return;
          }
          if (key === 'brandSingle') {
            _.each(edge.node[key], item => {
              const category = _.split(item.slug, '/')[1];

              queries.brandSingle(graphql).then(result => {
                if (result.errors) {
                  reject(result.errors);
                }

                const brandSingleTemplate = path.resolve(`./src/templates/brandSingle.js`);
                const filteredArray = _.filter(result.data.allContentfulMerk.edges, edge => {
                  let slug = _.get(edge.node, 'category.slug');
                  return _.includes(slug, category) && edge.node.node_locale === language;
                });

                _.each(filteredArray, edge => {
                  createPage({
                    path: `${language}/${_.replace(item.slug, ':slug', edge.node.slug)}`,
                    component: slash(brandSingleTemplate),
                    context: {
                      locale: language,
                      id: edge.node.id,
                      navId: `c1DzxdxolMIwGMs2wMucG8W${language === 'nl-BE'
                        ? ''
                        : `___${language}`}`,
                    },
                  });
                });
              });
              allPagesCreated.check('brandSingle');
            });
            return;
          }
          // if (key === 'horeca') {
          //   allPagesCreated.check('horeca');
          //   return;
          // }
          const template = path.resolve(`./src/templates/${key}.js`);
          if (edge.node[key]) {
            _.each(edge.node[key], item => {
              createPage({
                path: `/${item.node_locale}/${item.slug}`,
                component: slash(template),
                context: {
                  locale: item.node_locale,
                  navId: `c1DzxdxolMIwGMs2wMucG8W${item.node_locale === 'nl-BE'
                    ? ''
                    : `___${item.node_locale}`}`,
                 slug: item.slug,
                },
              });

              allPagesCreated.check(key);
            });
          }
        }
      });
    });
  });
}

exports.createPages = ({ graphql, boundActionCreators }) => {
  const { createPage } = boundActionCreators;

  return new Promise((resolve, reject) => {
    createPages(graphql, createPage, resolve, reject);
  });
};

// TODO dynamically create netlify redirect file: https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-plugin-netlify/src/create-redirects.js

exports.modifyWebpackConfig = ({ config, stage }) => {
  if (stage === 'build-html') {
    // Some libraries cannot be server side rendered.
    // More info: https://www.gatsbyjs.org/docs/debugging-html-builds/
    config.loader('null', {
      test: /react-flickity-component|ismobilejs|flickity-bg-lazyload|flatpickr|bowser|react-inview|flickity/,
      loader: 'null-loader',
    });
  }
};

gatsby-browser.js: not changed
gatsby-ssr.js: not changed

Actual result

What happened.

Expected behavior

What should happen?

Steps to reproduce

1.

2.

3.

...

[KyleAMathews]

We'll be doing a lot of optimizations to improve build performance in v2 and v3 of Gatsby but this is normal for now.

[hirviid]

Hi @KyleAMathews, do you have an idea on how much build performance will improve and what the expected timeline is for v2? What kind of improvements, something like incremental builds, so not all pages are rebuilt on every little change?

Currently 15 minutes is unacceptable for us (netlify stops the build after 15 minutes and will only increase if we add another language + customer is a little frustrated that just a small change takes so long before they can see it appear on their website). So we're looking into different solutions, maybe rewrite to a SSR solution like Next.js.

Any suggestion on this would be very much appreciated!
Thanks in advance!

[ricopags]

Would also like to know about anticipated pagebuild improvements, specifically what there is to be improved [mine is slow with many fewer than 900 pages, but even with verbose enabled I don't see why].

SUPER loving Gatsby, and as this is my first post in this GH I felt I should say so. Great project, looking forward to learning through it. Thanks for the great work!

[KyleAMathews]

Hey chaps — we merged a PR recently which speeds up builds AND reduces memory usage a lot #4555

v2 is getting close! Hopefully next week we'll get the first beta out + an upgrade guide.

[szimek]

@KyleAMathews Is it really likely to have a usable beta of v2 next week? We plan to release our blog next week, but we're running into build issues as well.

It currently takes ~30 minutes to build our blog (~4000+ pages + sitemaps, ~700 pages per language, 6 languages) on my macbook pro from 2014. It only works if I set NODE_OPTIONS=--max-old-space-size=6144, which is also an issue, because currently all our Jenkins nodes, which we use to build Docker images, have only 2GB of memory :/

There's also something wrong with either Contentful source plugin or source and transform nodes step in Gatsby. I'm fetching data from 6 different spaces (projects) in Contentful and all spaces have exactly the same content type (schema), just different language. With empty cache each consecutive space takes longer to process by Gatsby than the previous one - with 1 space it takes ~19s, with 2 it takes ~60s, with 6 spaces it takes ~555s, so as you can see it's not linear. More details here. Maybe it's already fixed in v2 as well ;)

[chrism2671]

What's the current status of this? We're considering a move to Gatsby, and have a site with ~1000 pages. If it's the wrong tool for the job, what should we be considering?

[KyleAMathews]

@chrism2671 check out v2! With the memory reduction PR I linked to earlier + my "hulksmash" PR, builds in v2 are ~80% faster for larger sites. #6226 (comment)

A 1000 page site should easily build in < 1 minute.