robots.txt + sitemap.xml

.gitignore

@@ -8,7 +8,6 @@ private*.json
 /node_modules
 /bower_components
 /modules/hudozka-api-v1/lib
-/public
 /build
 /out
 dist

next.config.js

@@ -1,6 +1,7 @@
 const withCss = require('@zeit/next-css')
 const withSass = require('@zeit/next-sass')
 const { range } = require('lodash')
+const fs = require('fs').promises
 
 function createPath(page, map, query, items) {
     return items.reduce((acc, path) => {
@@ -22,6 +23,8 @@ module.exports = withCss(withSass({
         // secondSecret: process.env.SECOND_SECRET // Pass through env variables
     },
     exportPathMap: async (defaultPathMap, { dev }) => {
+        await fs.copyFile('./public/robots.txt', './dist/robots.txt')
+        
         if (dev) {
             return null
         }

package.json

@@ -9,7 +9,8 @@
     "dev": "node src/server.js",
     "next": "next",
     "build": "next build",
-    "export": "npm run build && next export -o dist",
+    "export": "npm run build && next export -o dist && npm run sitemap",
+    "sitemap": "node scripts/sitemap.js",
     "gulp": "gulp",
     "test": "mocha --compilers js:babel-register",
     "coverage": "istanbul cover mocha -- --compilers js:babel-register"

scripts/sitemap.js

@@ -0,0 +1,80 @@
+const sitemap = require('sitemap')
+const axios = require('axios')
+const fs = require('fs').promises
+const homeUrl = 'https://art.shlisselburg.org'
+const sitemapCacheTime = 600000
+
+const api = axios.create({
+    baseURL: 'http://localhost:3000',
+})
+
+async function getMenuUrls(frequency = 'daily') {
+    const menu = [
+        '/',
+        '/schedule',
+        '/documents',
+        '/collective',
+        '/gallery',
+    ]
+    return menu
+        // .filter(i => 'url' in i)
+        .map(url => ({
+            url,
+            changefreq: frequency
+        }))
+}
+
+async function getPageUrls(frequency = 'daily') {
+    const res = await api.get('/api/pages/urls')
+
+    return res.data.items.map(url => ({
+        changefreq: frequency,
+        url,
+    }))
+}
+
+async function getAlbumsUrls() {
+    const res = await api.get('/api/albums/urls')
+    return res.data.items.map(url => ({
+        changefreq: 'monthly',
+        url,
+    }))
+}
+
+async function getTeacherUrls() {
+    const res = await api.get('/api/persons/urls')
+
+    return res.data.items.map(url => ({
+        changefreq: 'monthly',
+        url,
+    }))
+}
+
+async function getArticleUrls() {
+    const res = await api.get('/api/articles/urls')
+
+    return res.data.items.map(url => ({
+        changefreq: 'monthly',
+        url,
+    }))
+}
+
+(async () => {
+    let urls = await Promise.all([
+        getMenuUrls(),
+        getPageUrls(),
+        getAlbumsUrls(),
+        getTeacherUrls(),
+        getArticleUrls(),
+    ])
+    urls = urls.reduce((urls, i) => urls.concat(i))
+
+    let map = sitemap.createSitemap({
+        hostname: homeUrl,
+        cacheTime: sitemapCacheTime,
+        urls
+    })
+    const content = map.toString()
+
+    await fs.writeFile('dist/sitemap.xml', content, { encoding: 'utf-8' })
+})()

src/routes/sitemap.js

@@ -1,86 +0,0 @@
-const sitemap = require('sitemap')
-const route = require('koa-route')
-const {c} = require('../core/db')
-const {getPageUrls: pages} = require('../core/pages')
-const getPathWithNoTrailingSlash = require('../lib/url').getPathWithNoTrailingSlash
-
-const {homeUrl, sitemapCacheTime} = require('../config')
-const menu = require('../models/menu').default
-
-module.exports = function () {
-	return route.get('/sitemap.xml', async ctx => {
-		let urls = await Promise.all([
-			getMenuUrls(),
-			getPageUrls(),
-			getGalleryUrls(),
-			getTeacherUrls(),
-			getArticleUrls(),
-		])
-		urls = urls.reduce((urls, i) => urls.concat(i))
-
-		let map = sitemap.createSitemap({
-			hostname: homeUrl,
-			cacheTime: sitemapCacheTime,
-			urls: urls
-		})
-
-		ctx.set('Content-Type', 'application/xml')
-		ctx.body = map.toString()
-	})
-}
-
-async function getMenuUrls(frequency = 'daily') {
-	return menu
-		.filter(i => 'url' in i)
-		.map(i => ({
-			url: getPathWithNoTrailingSlash(i.url),
-			changefreq: frequency
-		}))
-}
-
-async function getPageUrls(frequency = 'daily') {
-	const ps = await pages()
-	return ps.map(i => ({
-		url: i.url,
-		changefreq: frequency
-	}))
-}
-
-async function getGalleryUrls() {
-	let docs = await c('albums')
-		.find({})
-		.toArray()
-
-	return docs.map(i => {
-		return {
-			url: `/album/${i.id}`,
-			changefreq: 'monthly'
-		}
-	})
-}
-
-async function getTeacherUrls() {
-	let docs = await c('collective')
-		.find({})
-		.toArray()
-
-	return docs.map(i => {
-		return {
-			url: `/teacher/${i.id}`,
-			changefreq: 'monthly'
-		}
-	})
-}
-
-async function getArticleUrls() {
-	let docs = await c('articles')
-		.find({})
-		.toArray()
-
-	return docs.map(i => {
-		return {
-			url: `/article/${i.id}`,
-			changefreq: 'monthly'
-		}
-	})
-}