merge changes in active branch to development

.circleci/config.yml

@@ -67,11 +67,14 @@ jobs:
             - statistics.json
   prepare-nuxt:
     docker:
-      - image: cimg/node:20.15.0
+      - image: informatievlaanderen/oslo-specification-generator:multilingual-dev4.0
     steps:
       - checkout
       - attach_workspace:
           at: /tmp/workspace
+      - add_ssh_keys:
+          fingerprints:
+            - "81:37:0b:68:3b:e9:c9:64:b8:f2:1f:c9:03:46:74:b0"
       - run: bash $PWD/scripts/copy-to-nuxt.sh /tmp/workspace
       - persist_to_workspace:
           root: /tmp/workspace
@@ -134,13 +137,13 @@ jobs:
           name: Push results to 'standaarden' branch of OSLO-Standaardenregister
           working_directory: /tmp/OSLO-Standaardenregister
           command: |
+            ls ./content/standaarden/
             git config user.email "oslo@oslo"
             git config user.name "Circle CI Builder"
             git add .
             git status
             git commit -m "Applying changes from commit ${CIRCLE_SHA1}" --allow-empty
             export TAG=`echo "${CIRCLE_SHA1}" | cut -c1-15`
-            git tag "${TAG}"
             git push --force origin standaarden
   build-state:
     docker:
@@ -176,6 +179,13 @@ jobs:
       - add_ssh_keys:
           fingerprints:
             - "15:51:57:05:3f:2c:a3:23:44:2a:52:5a:a6:aa:4c:c5"
+      - run:
+          name: download translation memory
+          working_directory: /tmp/workspace
+          command: |
+            export CIRCLEWKD=$( eval echo "${CIRCLE_WORKING_DIRECTORY}" )
+            GENERATEDREPO=$(jq --arg bt "${CIRCLE_BRANCH}" -r '.generatedrepository + {"filepath":"nuxtmemory.tgz", "branchtag":"\($bt)"}' ${CIRCLEWKD}/config/config.json)
+            if  ${CIRCLEWKD}/scripts/downloadFileGithub.sh "${GENERATEDREPO}" nuxtmemory.tgz ${TOOLCHAIN_TOKEN} ; then tar -xzf nuxtmemory.tgz ; fi
       - run:
           name: Checkout target repository
           working_directory: /tmp
@@ -205,6 +215,12 @@ jobs:
             if [ -d html_pages/geregistreerde-standaard ] ; then if [ ! "$(ls -A html_pages/geregistreerde-standaard)" ] ; then echo "Directory html_pages/geregistreerde-standaard is empty" ; else cp -R html_pages/geregistreerde-standaard/* /tmp/OSLO-StandaardenregisterGenerated/public/geregistreerde-standaard/ ; fi fi
             if [ -d html_pages/standaard-in-ontwikkeling ] ; then if [ ! "$(ls -A html_pages/standaard-in-ontwikkeling)" ] ; then echo "Directory html_pages/standaard-in-ontwikkeling is empty" ; else cp -R html_pages/standaard-in-ontwikkeling/* /tmp/OSLO-StandaardenregisterGenerated/public/standaard-in-ontwikkeling/ ; fi fi
             if [ -d html_pages/zonder-status ] ; then if [ ! "$(ls -A html_pages/zonder-status)" ] ; then echo "Directory html_pages/zonder-status is empty" ; else cp -R html_pages/zonder-status/* /tmp/OSLO-StandaardenregisterGenerated/public/zonder-status/; fi fi
+      - run:
+          name: Create new autotranslate memory of the nuxt directory
+          working_directory: /tmp/workspace
+          command: |
+            ls -1
+            if [ -d nuxtmemory ] ; then tar -zcf nuxtmemory.tgz nuxtmemory ; cp nuxtmemory.tgz /tmp/OSLO-StandaardenregisterGenerated/nuxtmemory.tgz ; fi
       - run:
           name: Insert the commit
           command: |
@@ -253,3 +269,4 @@ workflows:
 #            - checkout
 #            - prepare-nuxt
 #            - push-to-nuxt
+

Dockerfile.build

@@ -9,6 +9,9 @@ RUN rm -rf /app/content/standaarden
 # Copy all items from the content/standaarden directory
 COPY content/standaarden /app/content/standaarden
 
+# Copy the statistics file from content directory
+COPY content/statistics.json /app/content/statistics.json
+
 RUN npm run build
 
 FROM node:20-alpine

Makefile

@@ -14,7 +14,7 @@ fetch-content:
     cd OSLO-Standaardenregister && \
     git checkout standaarden && \
     ls && \
-    cp -R ./content/standaarden/* ../content/standaarden/
+    cp -R ./content/* ../content/
 	rm -rf OSLO-Standaardenregister
 
 # first build-base should have been run

scripts/contributors-and-terms.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+ROOTDIR=$1
+
+# Fetch the content from the URL
+content=$(curl -s "https://raw.githubusercontent.com/Informatievlaanderen/data.vlaanderen.be-statistics/production/aggr.stat")
+
+TOTAL_TERMS=$(echo "$content" | jq -r '.totalterms')
+
+echo "$TOTAL_TERMS"
+
+# Query certain keys and concatenate them into a JSON object
+uniqueContributors=$(echo "$content" | jq -r '(.authors | tonumber) + (.editors | tonumber) + (.contributors | tonumber) + (.participants | tonumber)')
+json=$(echo "$content" | jq -r --argjson uc "$uniqueContributors" '{uniqueContributors: $uc}')
+
+uniqueAffiliations=$(echo "$content" | jq -r '.totalorganisations | tonumber')
+json=$(echo "$json" | jq -r --argjson to "$uniqueAffiliations" '. + {uniqueAffiliations: $to}')
+
+
+# Read the existing content of statistics.json into a variable
+existingContent=$(cat "$ROOTDIR/statistics.json")
+
+# Add the new keys to the existing content
+updatedContent=$(echo "$existingContent" "$json" | jq -s add)
+
+# Write the updated content back to statistics.json
+echo "$updatedContent" >"$ROOTDIR/statistics.json"

scripts/copy-to-nuxt.sh

@@ -3,6 +3,11 @@
 ROOTDIR=$1
 REPODIR=$ROOTDIR/repositories
 NUXTDIR=$ROOTDIR/nuxt
+NUXTMEMORYDIR=$ROOTDIR/nuxtmemory
+
+# Define list of possible languages
+LANGUAGES=("en" "nl" "fr" "de")
+LANGUAGE_STRING=$(IFS=','; echo "${LANGUAGES[*]}")
 
 mkdir -p "$NUXTDIR"
 
@@ -21,25 +26,83 @@ do
 
 
   ## this needs to change to .title if the new json structure is used
-  SPEC_NAME=$(jq -r '.naam' "$CONFIG_NAME.json")
+  SPEC_NAME=$(jq -r '.title' "$CONFIG_NAME.json")
   ## this needs to change to .descriptionFileName if the new json structure is used
-  DESCRIPTION_NAME=$(jq -r '.beschrijving' "$CONFIG_NAME.json")
+  DESCRIPTION_NAME=$(jq -r '.descriptionFileName' "$CONFIG_NAME.json")
 
   echo "SPEC_NAME: $SPEC_NAME"
   echo "DESCRIPTION_NAME: $DESCRIPTION_NAME"
 
   ## Normalizing spec name to be used as directory name
   NORMALIZED_SPEC_NAME="$(echo $SPEC_NAME | iconv -f utf8 -t ascii//TRANSLIT | tr -c '[:alnum:]\n\r' '-' | tr -s '-' | tr '[:upper:]' '[:lower:]')"
+  NUXTMEMORYNORMALIZED_SPEC_NAME="$NUXTMEMORYDIR/$NORMALIZED_SPEC_NAME"
 
   echo "NORMALIZED_SPEC_NAME: $NORMALIZED_SPEC_NAME"
 
   mkdir -p "$NUXTDIR/$NORMALIZED_SPEC_NAME"
+  mkdir -p "$NUXTMEMORYNORMALIZED_SPEC_NAME"
 
   # Store the repository name in the configuration file
   jq --arg REPOSITORY "$REPO_NAME" '. + {"repository": $REPOSITORY}' "$CONFIG_NAME.json" > "temp.json" && mv "temp.json" "$CONFIG_NAME.json"
 
-  ## Copy the generated configuration file and description file to the nuxt directory
-  cp "$CONFIG_NAME.json" "$NUXTDIR/$NORMALIZED_SPEC_NAME/configuration.json"
+  CONFIGMD5SUM="$NUXTMEMORYNORMALIZED_SPEC_NAME/nl/configuration.json.md5sum"
+  if [ -f "$NUXTMEMORYNORMALIZED_SPEC_NAME/nl/configuration.json" ]; then
+    # Check if md5sum is the same
+    CURSUM=$(md5sum "$CONFIG_NAME.json")
+    OLDSUM=$(cat "$CONFIGMD5SUM")
+    if [ "$CURSUM" == "$OLDSUM" ]; then
+       echo "Use old configuration file"
+       for lang in "${LANGUAGES[@]}"; do
+         cp "$NUXTMEMORYNORMALIZED_SPEC_NAME/$lang/configuration.json" "$NUXTDIR/$NORMALIZED_SPEC_NAME/$lang/configuration.json"
+       done
+    else
+      translate_and_copy_config
+    fi
+  else
+    translate_and_copy_config
+  fi  
+
+  # Check if the description file exists and didn't change in the memory
   cp "descriptions/$DESCRIPTION_NAME" "$NUXTDIR/$NORMALIZED_SPEC_NAME/description.md"
-
-done < "$ROOTDIR/tmp-register.txt"
+  DESCRIPTIONFILE="$NUXTDIR/$NORMALIZED_SPEC_NAME/description.md"
+  MD5SUMFILE="$NUXTMEMORYNORMALIZED_SPEC_NAME/nl/description.md.md5sum"
+  # Check if file is in memory
+  if [ -f "$NUXTMEMORYNORMALIZED_SPEC_NAME/nl/descriptions.md" ]; then
+    # Check if md5sum is the same
+    CURSUM=$(md5sum "$DESCRIPTIONFILE")
+    OLDSUM=$(cat "$MD5SUMFILE")
+    if [ "$CURSUM" == "$OLDSUM" ]; then
+       echo "Use old description file"
+       for lang in "${LANGUAGES[@]}"; do
+         cp "$NUXTMEMORYNORMALIZED_SPEC_NAME/$lang/description.md" "$NUXTDIR/$NORMALIZED_SPEC_NAME/$lang/description.md"
+       done
+    else
+      md5sum "$DESCRIPTIONFILE" > "$MD5SUMFILE"
+      node /app/autotranslate-md.js -i "$DESCRIPTIONFILE" -m "nl" -g "$LANGUAGE_STRING" -s "$AZURETRANSLATIONKEY"
+    fi
+  else
+    md5sum "$DESCRIPTIONFILE" > "$MD5SUMFILE"
+    node /app/autotranslate-md.js -i "$DESCRIPTIONFILE" -m "nl" -g "$LANGUAGE_STRING" -s "$AZURETRANSLATIONKEY"
+  fi
+
+  rm "$NUXTDIR/$NORMALIZED_SPEC_NAME/description.md"
+
+  # Copy the translations to the memory
+  cp -r "$NUXTDIR/$NORMALIZED_SPEC_NAME/"* "$NUXTMEMORYNORMALIZED_SPEC_NAME"
+
+done < "$ROOTDIR/tmp-register.txt"
+
+translate_and_copy_config() {
+  md5sum "$CONFIG_NAME.json" > "$CONFIGMD5SUM"
+  node /app/autotranslate-config.js -i "$CONFIG_NAME.json" -o "$CONFIG_NAME-multilang.json" -m "nl" -g "$LANGUAGE_STRING" -s "$AZURETRANSLATIONKEY"
+  cp "$CONFIG_NAME-multilang.json" "$NUXTDIR/$NORMALIZED_SPEC_NAME/configuration.json"
+  ## Convert config file to single language and copy the generated configuration file to the nuxt directory
+  for lang in "${LANGUAGES[@]}"; do
+    mkdir -p "$NUXTDIR/$NORMALIZED_SPEC_NAME/$lang"
+    if ! node /app/convert-config.js -i "$CONFIG_NAME-multilang.json" -l "$lang" -o "$NUXTDIR/$NORMALIZED_SPEC_NAME/$lang/configuration.json"; then
+      echo "Convert config: failed"
+    else
+      echo "Convert config: Files successfully translated"
+    fi
+  done
+}

scripts/cronjob

@@ -1,4 +1,3 @@
-# THIS IS THE CRONJOB THAT SHOULD RUN ON THE SERVER TO CHECK FOR NEW IMAGES AND UPDATE THE SERVICE IF NEEDED
 
 #!/bin/bash
 
@@ -9,7 +8,12 @@ echo "Cron job run at: $(date)"
 IMAGE_NAME="terraformtestcontainerregistry.azurecr.io/oslo2/oslo-standaardenregister-run"
 
 # Define an array of service names
-SERVICE_NAMES=("oslo_dev-standardsregister" "oslo_test-standardsregister")
+SERVICE_NAMES=("oslo_test-standardsregister" "oslo_dev-standardsregister")
+
+# Pull the latest version of the image
+docker pull $IMAGE_NAME:latest
+
+LATEST_IMAGE_SHA=$(docker inspect --format='{{index .RepoDigests 0}}' $IMAGE_NAME:latest | perl -wnE'say /sha256.*/g')
 
 # Loop over the array of service names
 for SERVICE_NAME in "${SERVICE_NAMES[@]}"; do
@@ -19,11 +23,6 @@ for SERVICE_NAME in "${SERVICE_NAMES[@]}"; do
   CURRENT_IMAGE=$(docker service inspect --format '{{ (.Spec.TaskTemplate.ContainerSpec.Image) }}' $SERVICE_NAME)
   CURRENT_IMAGE_SHA=$(docker inspect --format='{{.Id}}' $CURRENT_IMAGE)
 
-  # Pull the latest version of the image
-  docker pull $IMAGE_NAME:latest
-
-  LATEST_IMAGE_SHA=$(docker inspect --format='{{.Id}}' $IMAGE_NAME:latest)
-
   echo $CURRENT_IMAGE_SHA
   echo $LATEST_IMAGE_SHA
 

scripts/extract-info.sh

@@ -8,8 +8,7 @@ REPODIR=$ROOTDIR/repositories
 ls "$ROOTDIR"
 
 ## Constructing file names for description
-while IFS= read -r line
-do
+while IFS= read -r line; do
   REPO_NAME=$(echo "$line" | cut -d ":" -f 1)
 
   cd "$REPODIR/$REPO_NAME"
@@ -19,20 +18,20 @@ do
   ## Extract information about the files containing the description for the detail page
   DESCRIPTION_NAME=$(jq -r '.beschrijving' "$CONFIG_NAME.json")
 
-  echo "$CONFIG_NAME:$DESCRIPTION_NAME" >> "$ROOTDIR/description-paths.txt"
+  echo "$CONFIG_NAME:$DESCRIPTION_NAME" >>"$ROOTDIR/description-paths.txt"
 
   #### Constructing file name ####
   SPECIAL_CHARACTERS_REMOVED="${SPEC_NAME//[:&]/}"
   MULTI_SPACE_REMOVED="${SPECIAL_CHARACTERS_REMOVED//  /}"
   SPACE_REPLACED="${MULTI_SPACE_REMOVED// /-}"
 
-  echo "$CONFIG_NAME:$SPACE_REPLACED" >> "$ROOTDIR/filenames.txt"
-done < "$ROOTDIR/tmp-register.txt"
+  echo "$CONFIG_NAME:$SPACE_REPLACED" >>"$ROOTDIR/filenames.txt"
+done <"$ROOTDIR/tmp-register.txt"
 
 ## Creating statistics config file
 echo "Creating statistics configuration file"
 touch "$ROOTDIR/statistics_config.json"
-echo "[]" > "$ROOTDIR/statistics_config.json"
+echo "[]" >"$ROOTDIR/statistics_config.json"
 
 ## Constructing statistics configuration file
 if cat "$REGISTER" | jq -e . >/dev/null 2>&1; then
@@ -44,7 +43,6 @@ if cat "$REGISTER" | jq -e . >/dev/null 2>&1; then
     REPOSITORY=$(_jq '.repository')
     THEME_NAME=$(echo "$REPOSITORY" | cut -d '/' -f 5)
     CONFIG=$(_jq '.configuration')
-    STATUS=$(_jq '.status')
 
     if [ ! -d "$REPODIR/$THEME_NAME" ]; then
       git clone "$REPOSITORY" "$REPODIR/$THEME_NAME"
@@ -53,16 +51,11 @@ if cat "$REGISTER" | jq -e . >/dev/null 2>&1; then
     cd "$REPODIR/$THEME_NAME"
     git checkout standaardenregister
 
-    NAME=$(cat "$CONFIG" | jq -r '.naam')
-    REPORT_FILE=$(cat "$CONFIG" | jq -r '.rapport')
-    PUB_DATE=$(cat "$CONFIG" | jq -r ".publicatiedatum")
+    NAME=$(cat "$CONFIG" | jq -r '.title')
+    PUB_DATE=$(cat "$CONFIG" | jq -r ".publicationDate")
+    STATUS=$(cat "$CONFIG" | jq -r ".status")
 
-
-    if [ -z "$REPORT_FILE"  ]; then
-      jq --arg REPOSITORY "$THEME_NAME" --arg NAAM "$NAME" --arg STATUS "$STATUS" --arg REPORT "$REPORT_FILE" --arg PUB_DATE "$PUB_DATE" '. += [{"name": $NAAM, "repository": $REPOSITORY, "report" : null, "status" : $STATUS, "publicationDate" : $PUB_DATE}]' "$ROOTDIR/statistics_config.json" > "$ROOTDIR/statistics_config.json.tmp" && mv "$ROOTDIR/statistics_config.json.tmp" "$ROOTDIR/statistics_config.json"
-    else
-      jq --arg REPOSITORY "$THEME_NAME" --arg NAAM "$NAME" --arg STATUS "$STATUS" --arg REPORT "$REPORT_FILE" --arg PUB_DATE "$PUB_DATE" '. += [{"name": $NAAM, "repository": $REPOSITORY, "report" : $REPORT, "status" : $STATUS, "publicationDate" : $PUB_DATE}]' "$ROOTDIR/statistics_config.json" > "$ROOTDIR/statistics_config.json.tmp" && mv "$ROOTDIR/statistics_config.json.tmp" "$ROOTDIR/statistics_config.json"
-    fi
+    jq --arg REPOSITORY "$THEME_NAME" --arg NAAM "$NAME" --arg STATUS "$STATUS" --arg PUB_DATE "$PUB_DATE" '. += [{"name": $NAAM, "repository": $REPOSITORY, "status" : $STATUS, "publicationDate" : $PUB_DATE}]' "$ROOTDIR/statistics_config.json" >"$ROOTDIR/statistics_config.json.tmp" && mv "$ROOTDIR/statistics_config.json.tmp" "$ROOTDIR/statistics_config.json"
   done
 fi
 

scripts/sanitize-configurations.sh

@@ -2,5 +2,6 @@
 
 JS_FILE_PATH="scripts/sanitizeConfigurations.js"
 
+
 # Use Node.js to run the JavaScript file
 node $JS_FILE_PATH