New WebFu Info Architecture (google#5003)

* New sitewide footer

* New master project.yaml

* Updated feedback section

* updated styles for feedback section

* Updated tools section

* Updates localized tools to use tools/_project.yaml

* fixes duplicate mapping key

* Moves codelabs to /web/fu/codelabs

* updates test suite for current behaviour

* longer what’s new, plus landing page images

* moves app install banners

* architecture & web components

* principles & remove d&u index

* Moves discovery

* moving engage-and-retain

* payments, push, discovery

* primers, payments, wsk

* integration, native hardware

* more file moves

* link fixes

* sets project page to /web/fund/proj.yaml

* wip

* design-and-ui to ux

* WIP

* WIP

* properly parse book.yaml file

* misc bug fixes

* don’t report warnings temporarily

* broken redirect fixes

* fix urls in web components

* broken link fixes

* broken link fixes

* broken link fixes

* broken link fixes

* broken link fixes

* Style guide updates

* parse proper project file

* fixes duplicate calls to read book.yaml

* add test to verify includecode widget finds file

* fixes broken {% includecode %} links

* broken link fixes

* remove the glossary file

* automatic building of the glossary

* use abbr instead of acronym

* new FIRE images & more glossary definitions

* more glossary items

* fixing localized content

* move l10n files

* find orphaned l10n files

* removing orphaned files

* moving l10n files

* URL fixes

* autobuild announcement

* broken link fixes

* glossary style updates

* fix ordering on TOC

* additional links for WorkBox

* announcement tweak

* "High-performance service worker loading" article (google#5049)

.gitignore

@@ -14,6 +14,7 @@ src/content/**/*/_files.json
 src/content/**/*/atom.xml
 src/content/**/*/rss.xml
 src/content/**/*/feed.xml
+src/content/*/fundamentals/glossary.md
 src/content/*/_shared/contributors/*.html
 src/content/*/_shared/latest_show.html
 src/content/*/_shared/latest_articles.html

devsiteHelper.py

@@ -12,6 +12,9 @@
 from google.appengine.ext.webapp.template import render
 
 SOURCE_PATH = os.path.join(os.path.dirname(__file__), 'src/content/')
+DEVENV = os.environ['SERVER_SOFTWARE'].startswith('Dev')
+USE_MEMCACHE = not DEVENV
+
 
 def slugify(str):
   # Very simply slugify
@@ -20,18 +23,22 @@ def slugify(str):
   slug = re.sub(r'[-]+', '-', slug)
   return slug
 
+
 def getFromMemCache(memcacheKey):
   try:
     result = memcache.get(memcacheKey)
   except Exception as e:
     result = None
   return result
 
-def setMemCache(memcacheKey, value):
+
+def setMemCache(memcacheKey, value, length=3600):
   try:
-    memcache.set(memcacheKey, value)
+    if USE_MEMCACHE:
+      memcache.set(memcacheKey, value, length)
   except Exception as e:
-    pass
+    logging.exception('Unable to cache to MemCache')
+
 
 def checkForRedirect(requestedPath, lang, useMemcache):
   # Reads the redirect files from the current directory and up the directory
@@ -88,7 +95,6 @@ def checkForRedirect(requestedPath, lang, useMemcache):
 def readFile(requestedFile, lang='en'):
   # Reads a file from the file system, first trying the localized, then
   # the English version. If neither exist, it returns None
-  #originalPathToFile = pathToFile
   requestedFile = re.sub(r'^/?web/', '', requestedFile)
   workingFile = os.path.join(SOURCE_PATH, lang, requestedFile)
   if not os.path.isfile(workingFile):
@@ -107,6 +113,7 @@ def readFile(requestedFile, lang='en'):
     logging.error(result)
     return None
 
+
 def fetchGithubFile(path):
   """Fetchs a file in a repository hosted on github.com.
 
@@ -129,35 +136,117 @@ def fetchGithubFile(path):
   except urllib2.HTTPError as e:
     logging.error('Unable to fetch Github file: %s' % e.code)
     return None
-
   return content
 
-def getLeftNav(requestPath, pathToBook, lang='en'):
+
+def parseBookYaml(pathToBook, lang='en'):
+  """Read and parse a book.yaml file.
+
+  Args:
+      pathToBook: the string path to the location of the book
+      lang: Which language to use, defaults to 'en'
+
+  Returns:
+      A dictionary with the parsed book.
+  """
+  memcacheKey = 'bookYAML-' + pathToBook
+  result = getFromMemCache(memcacheKey)
+  if result:
+    return result
+  try:
+    result = {}
+    upperTabs = []
+    result['upper_tabs'] = upperTabs
+    bookYaml = yaml.load(readFile(pathToBook, lang))
+    for upperTab in bookYaml['upper_tabs']:
+      upperTabs.append(expandBook(upperTab))
+    setMemCache(memcacheKey, result, 60)
+    return result
+  except Exception as e:
+    logging.exception('Error in parseBookYaml')
+  return None
+
+
+def expandBook(book, lang='en'):
+  """Iterate and expand includes in a book.yaml file.
+
+  Args:
+      book: the parsed book.yaml file
+      lang: Which language to use, defaults to 'en'
+
+  Returns:
+      A dictionary with the parsed & expanded book.
+  """
+  if isinstance(book, dict):
+    result = {}
+    for k, v in book.iteritems():
+      result[k] = expandBook(v, lang)
+    return result
+  if isinstance(book, list):
+    results = []
+    for item in book:
+      if 'include' in item:
+        newItems = yaml.load(readFile(item['include'], lang))
+        results = results + expandBook(newItems['toc'], lang)
+      else:
+        results.append(expandBook(item, lang))
+    return results
+  return book
+
+
+def getLowerTabs(bookYaml):
+  """Gets the lower tabs from a parsed book.yaml dictionary.
+
+  Args:
+      bookYaml: the parsed book.yaml file
+
+  Returns:
+      An array of objects with the lower tabs
+  """
+  result = []
+  try:
+    for tab in bookYaml['upper_tabs']:
+      if 'lower_tabs' in tab and 'other' in tab['lower_tabs']:
+        for lowerTab in tab['lower_tabs']['other']:
+          lt = {}
+          lt['name'] = lowerTab['name']
+          if 'contents' in lowerTab and 'path' in lowerTab['contents'][0]:
+            lt['path'] = lowerTab['contents'][0]['path']
+            result.append(lt)
+  except Exception as e:
+    logging.exception('Unable to read/parse the lower tabs')
+  return result
+
+
+def getLeftNav(requestPath, bookYaml, lang='en'):
   # Returns the left nav. If it's already been generated and stored in
   # memcache, return that, otherwise, read the file then recursively
   # build the tree using buildLeftNav.
+  memcacheKey = 'leftNav-' + requestPath
+  result = getFromMemCache(memcacheKey)
+  if result:
+    return result
   whoops = '<h2>Whoops!</h2>'
   whoops += '<p>An error occured while trying to parse and build the'
   whoops += ' left hand navigation. Check the error logs.'
   whoops += '</p>'
   requestPath = os.path.join('/web/', requestPath)
-  bookContents = readFile(pathToBook, lang)
-  if bookContents:
-    try:
-      yamlNav = yaml.load(bookContents)
-      for tab in yamlNav['upper_tabs']:
-        if 'path' in tab and requestPath.startswith(tab['path']):
-          if 'lower_tabs' in tab:
-            result = '<ul class="devsite-nav-list devsite-nav-expandable">\n'
-            result += buildLeftNav(tab['lower_tabs']['other'][0]['contents'])
-            return result
-    except Exception as e:
-      msg = ' - Unable to read or parse primary book.yaml: ' + pathToBook
-      logging.exception(msg)
-      whoops += '<p>Exception occured.</p>'
-      return whoops
-  else:
-    whoops += '<p>Not found: ' + pathToBook + '</p>'
+  try:
+    result = '<h2>No Matches Found</h2>'
+    for upperTab in bookYaml['upper_tabs']:
+      if 'path' in upperTab and requestPath.startswith(upperTab['path']):
+        for lowerTab in upperTab['lower_tabs']['other']:
+          if ('path' not in lowerTab['contents'][0] or
+            requestPath.startswith(lowerTab['contents'][0]['path'])):
+              result = '<ul class="devsite-nav-list devsite-nav-expandable">\n'
+              result += buildLeftNav(lowerTab['contents'])
+              result += '</ul>\n'
+    setMemCache(memcacheKey, result)
+    return result
+  except Exception as e:
+    msg = ' - Unable to read or parse primary book.yaml: ' + pathToBook
+    logging.exception(msg)
+    whoops += '<p>Exception occured.</p>'
     return whoops
 
 
@@ -166,15 +255,17 @@ def buildLeftNav(bookYaml, lang='en'):
   result = ''
   for item in bookYaml:
     if 'include' in item:
-      include = readFile(item['include'], lang)
-      if include:
-        try:
-          include = yaml.load(include)
-          if 'toc' in include and len(include['toc']) == 1:
-            item = include['toc'][0]
-        except Exception as e:
-          msg = ' - Unable to parsing embedded toc file: ' + item['include']
-          logging.exception(msg)
+      ## TODO(petele): Remove this
+      ## leaving this in for a few weeks while I ensure it doesn't break
+      ## anything.
+      logging.error('***** INCLUDE - this should NOT happen.')
+      # try:
+      #   include = readFile(item['include'], lang)
+      #   include = yaml.load(include)
+      #   result += buildLeftNav(include['toc'])
+      # except Exception as e:
+      #   msg = ' - Unable to parsing embedded toc file: ' + item['include']
+      #   logging.exception(msg)
     if 'path' in item:
       result += '<li class="devsite-nav-item">\n'
       result += '<a href="' + item['path'] + '" class="devsite-nav-title">\n'
@@ -196,7 +287,7 @@ def buildLeftNav(bookYaml, lang='en'):
       result += '</a>'
       result += '<ul class="devsite-nav-section devsite-nav-section-collapsed">\n'
       result += buildLeftNav(item['section'])
-  result += '</ul>\n'
+      result += '</ul>\n'
   return result
 
 
@@ -350,17 +441,13 @@ def getIncludeCode(include_tag, lang='en'):
   return cgi.escape(result)
 
 
-def getAnnouncementBanner(lang='en'):
-  # Returns the announcement banner
-  memcacheKey = 'header-Announcement-' + lang
+def getAnnouncementBanner(pathToProject, lang='en'):
+  memcacheKey = 'projectYAML-' + pathToProject
   result = getFromMemCache(memcacheKey)
-  if result is None:
-    result = ''
-    projectFile = os.path.join(SOURCE_PATH, lang, '_project.yaml')
-    if not os.path.isfile(projectFile):
-      projectFile = os.path.join(SOURCE_PATH, 'en', '_project.yaml')
-    raw = open(projectFile, 'r').read().decode('utf8')
-    project = yaml.load(raw)
+  if result:
+    return result
+  try:
+    project = yaml.load(readFile(pathToProject, lang))
     if 'announcement' in project:
       startBanner = project['announcement']['start']
       startBanner = datetime.strptime(startBanner, '%Y-%m-%dT%H:%M:%SZ')
@@ -374,7 +461,10 @@ def getAnnouncementBanner(lang='en'):
         result += '</div>'
       else:
         logging.warn('Announcement in _project.yaml expired: not shown')
-      setMemCache(memcacheKey, result)
+      setMemCache(memcacheKey, result, 60)
+  except Exception as e:
+    logging.exception('Unable to get announcement from project.yaml')
+    return ''
   return result
 
 

devsiteIndex.py

@@ -143,12 +143,16 @@ def parseIndexYamlItems(yamlItems):
 def generateYaml(lang, requestPath, rawYaml):
   content = ''
   parsedYaml = yaml.load(rawYaml)
+  bookPath = parsedYaml['book_path']
+  bookYaml = devsiteHelper.parseBookYaml(bookPath, lang)
+  projectPath = parsedYaml['project_path']
   page = parsedYaml['landing_page']
   rows = page['rows']
   title = 'Web'
-  banner = devsiteHelper.getAnnouncementBanner(lang)
+  banner = devsiteHelper.getAnnouncementBanner(projectPath, lang)
   header = 'Generic Page Header Here'
   customCss = ''
+  lowerTabs = devsiteHelper.getLowerTabs(bookYaml)
   if 'custom_css_path' in page:
     customCss = '<link rel="stylesheet" href="'
     customCss += page['custom_css_path']
@@ -205,6 +209,7 @@ def generateYaml(lang, requestPath, rawYaml):
                 'title': title,
                 'announcementBanner': banner,
                 'requestPath': requestPath,
+                'lowerTabs': lowerTabs,
                 'customcss': customCss,
                 'header': header,
                 'content': content,

devsitePage.py

@@ -17,6 +17,7 @@ def getPage(requestPath, lang):
   title = 'Web Fundamentals'
   leftNav = '- No Left Nav Found - '
   toc = '- No TOC Found - '
+  announcementBanner = ''
   template = 'gae/article.tpl'
   fileLocations = [
     os.path.join(SOURCE_PATH, lang, requestPath) + '.md',
@@ -95,7 +96,13 @@ def getPage(requestPath, lang):
         # Reads the book.yaml file and generate the lefthand nav
         if 'book_path' in md.Meta and len(md.Meta['book_path']) == 1:
           bookPath = md.Meta['book_path'][0]
-          leftNav = devsiteHelper.getLeftNav(requestPath, bookPath, lang)
+          bookYaml = devsiteHelper.parseBookYaml(bookPath, lang)
+          leftNav = devsiteHelper.getLeftNav(requestPath, bookYaml)
+          lowerTabs = devsiteHelper.getLowerTabs(bookYaml)
+
+        if 'project_path' in md.Meta and len(md.Meta['project_path']) == 1:
+          projectPath = md.Meta['project_path'][0]
+          announcementBanner = devsiteHelper.getAnnouncementBanner(projectPath, lang)
 
         # Checks if the page should be displayed in full width mode
         if 'full_width' in md.Meta and len(md.Meta['full_width']) == 1:
@@ -131,12 +138,11 @@ def getPage(requestPath, lang):
       gitHubIssueUrl += lang + ']&body='
       gitHubIssueUrl += gitHubEditUrl
 
-      x = devsiteHelper.getFooterPromo()
-
       # Renders the content into the template
       response = render(template, {
         'title': title,
-        'announcementBanner': devsiteHelper.getAnnouncementBanner(lang),
+        'announcementBanner': announcementBanner,
+        'lowerTabs': lowerTabs,
         'gitHubIssueUrl': gitHubIssueUrl,
         'gitHubEditUrl': gitHubEditUrl,
         'requestPath': requestPath.replace('/index', ''),

gae/article.tpl

@@ -98,21 +98,13 @@
                 <div class="devsite-doc-set-nav-row devsite-full-site-width">
                   <nav class="devsite-doc-set-nav devsite-nav devsite-overflow-tabs-scroll-wrapper">
                     <ul class="devsite-doc-set-nav-tab-list devsite-overflow-tabs-scroll">
-                      <li class="devsite-doc-set-nav-tab-container">
-                        <a href="#" class="devsite-doc-set-nav-active devsite-doc-set-nav-tab">
-                          Tab 1
-                        </a>
-                      </li>
-                      <li class="devsite-doc-set-nav-tab-container">
-                        <a href="#" class="devsite-doc-set-nav-tab">
-                          Tab 1
-                        </a>
-                      </li>
-                      <li class="devsite-doc-set-nav-tab-container">
-                        <a href="#" class="devsite-doc-set-nav-tab">
-                          Tab 1
-                        </a>
-                      </li>
+                      {% for tab in lowerTabs %}
+                        <li class="devsite-doc-set-nav-tab-container">
+                          <a href="{{tab.path}}" class="devsite-doc-set-nav-tab">
+                            {{tab.name}}
+                          </a>
+                        </li>
+                      {% endfor %}
                     </ul>
                   </nav>
                 </div>

gae/home.tpl

@@ -88,21 +88,13 @@
                 <div class="devsite-doc-set-nav-row devsite-full-site-width">
                   <nav class="devsite-doc-set-nav devsite-nav devsite-overflow-tabs-scroll-wrapper">
                     <ul class="devsite-doc-set-nav-tab-list devsite-overflow-tabs-scroll">
-                      <li class="devsite-doc-set-nav-tab-container">
-                        <a href="#" class="devsite-doc-set-nav-active devsite-doc-set-nav-tab">
-                          Tab 1
-                        </a>
-                      </li>
-                      <li class="devsite-doc-set-nav-tab-container">
-                        <a href="#" class="devsite-doc-set-nav-tab">
-                          Tab 1
-                        </a>
-                      </li>
-                      <li class="devsite-doc-set-nav-tab-container">
-                        <a href="#" class="devsite-doc-set-nav-tab">
-                          Tab 1
-                        </a>
-                      </li>
+                      {% for tab in lowerTabs %}
+                        <li class="devsite-doc-set-nav-tab-container">
+                          <a href="{{tab.path}}" class="devsite-doc-set-nav-tab">
+                            {{tab.name}}
+                          </a>
+                        </li>
+                      {% endfor %}
                     </ul>
                   </nav>
                 </div>