Merge branch 'master' into sictiru

* master: (69 commits)
  iOS v13.1.1, fixing crash on iOS 16.
  Bumped version number
  Fixed crash on older iOS versions
  iOS v13.1
  Handling missing youtube duration.
  Fixing strip underscore
  iOS 15 screenshots
  Don't use the underscore cache buster anymore.
  Correct archive sub count
  Don't resync stories on new sub if it already has archive subscribers.
  More numpy changes.
  Strip underscores from feed addresses, special case jwz
  Load offline story images even when online to speed up image display.
  #1875 (borders between panes not using theme colors)
  #1247 (Mac Catalyst edition)
  #1874 (Crash on opening widget story)
  Allowing traffic on docker0 interface.
  Clean TXT records so letsencrypt doesn't fail
  Thresholding bounced newsletters email.
  #1247 (Mac Catalyst edition)
  ...

ansible/playbooks/setup_mongo.yml

@@ -6,18 +6,18 @@
     - motd_role: db
   vars_files:
     - ../env_vars/base.yml
-  
+
   roles:
-    - {role: 'base', tags: 'base'}
-    - {role: 'ufw', tags: 'ufw'}
-    - {role: 'docker', tags: 'docker'}
-    - {role: 'repo', tags: ['repo', 'pull']}
-    - {role: 'dnsmasq', tags: 'dnsmasq'}
-    - {role: 'consul', tags: 'consul'}
-    - {role: 'consul-client', tags: 'consul'}
-    - {role: 'mongo', tags: 'mongo'}
-    - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
-    - {role: 'mongo-exporter', tags: ['mongo-exporter', 'metrics']}
-    - {role: 'monitor', tags: 'monitor'}
-    - {role: 'flask_metrics', tags: ['flask-metrics', 'metrics']}
+    - { role: "base", tags: "base" }
+    - { role: "ufw", tags: "ufw" }
+    - { role: "docker", tags: "docker" }
+    - { role: "repo", tags: ["repo", "pull"] }
+    - { role: "dnsmasq", tags: "dnsmasq" }
+    - { role: "consul", tags: "consul" }
+    - { role: "consul-client", tags: "consul" }
+    - { role: "mongo", tags: "mongo" }
+    - { role: "node-exporter", tags: ["node-exporter", "metrics"] }
+    - { role: "mongo-exporter", tags: ["mongo-exporter", "metrics"] }
+    - { role: "monitor", tags: "monitor" }
+    - { role: "flask_metrics", tags: ["flask-metrics", "metrics"] }
     # - {role: 'benchmark', tags: 'benchmark'}

ansible/roles/consul/tasks/get_consul_manager_ip.py

@@ -14,24 +14,31 @@ def get_host_ips_from_group(group_name):
     :param inventory_base_path: Base path to the inventory directories. Defaults to the path in ansible.cfg.
     :return: A list of IP addresses belonging to the specified group.
     """
-    cmd = ['ansible-inventory', '-i', '/srv/newsblur/ansible/inventories/hetzner.ini', '-i', '/srv/newsblur/ansible/inventories/hetzner.yml', '--list']
-
+    cmd = [
+        "ansible-inventory",
+        "-i",
+        "/srv/newsblur/ansible/inventories/hetzner.ini",
+        "-i",
+        "/srv/newsblur/ansible/inventories/hetzner.yml",
+        "--list",
+    ]
+
     try:
         # Execute the ansible-inventory command
         result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True)
-        
+
         # Parse the JSON output from ansible-inventory
         inventory_data = json.loads(result.stdout)
-        
+
         host_ips = []
         # Check if the group exists
         if group_name in inventory_data:
             # Get the list of hosts in the specified group
-            if 'hosts' in inventory_data[group_name]:
-                for host in inventory_data[group_name]['hosts']:
+            if "hosts" in inventory_data[group_name]:
+                for host in inventory_data[group_name]["hosts"]:
                     # Fetch the host details, specifically looking for the ansible_host variable for the IP
-                    host_vars = inventory_data['_meta']['hostvars'][host]
-                    ip_address = host_vars.get('ansible_host', None)
+                    host_vars = inventory_data["_meta"]["hostvars"][host]
+                    ip_address = host_vars.get("ansible_host", None)
                     if ip_address:
                         host_ips.append(ip_address)
                     else:
@@ -50,16 +57,19 @@ def get_host_ips_from_group(group_name):
 
 with open(TOKEN_FILE) as f:
     token = f.read().strip()
-    os.environ['DO_API_TOKEN'] = token
+    os.environ["DO_API_TOKEN"] = token
 
 manager = digitalocean.Manager(token=token)
 my_droplets = manager.get_all_droplets()
 consul_manager_droplets = [d for d in my_droplets if "db-consul" in d.name]
 
 # Use ansible-inventory to get the consul-manager ip
-group_name = 'hconsul'
+group_name = "hconsul"
 hetzner_hosts = get_host_ips_from_group(group_name)
-consul_manager_ip_address = ','.join([f"\"{droplet.ip_address}\"" for droplet in consul_manager_droplets] + [f"\"{host}\"" for host in hetzner_hosts])
+consul_manager_ip_address = ",".join(
+    [f'"{droplet.ip_address}"' for droplet in consul_manager_droplets]
+    + [f'"{host}"' for host in hetzner_hosts]
+)
 
 print(consul_manager_ip_address)
 

ansible/roles/mongo/tasks/main.yml

@@ -49,7 +49,7 @@
 
     - name: Create the mount point
       become: yes
-      file: 
+      file:
         path: "/mnt/{{ inventory_hostname | regex_replace('db-|-', '') }}"
         state: directory
         owner: "{{ ansible_effective_user_id|int }}"
@@ -64,7 +64,6 @@
         opts: defaults,discard
         state: mounted
 
-
     - name: Set permissions on mongo volume
       # become: yes
       file:
@@ -93,7 +92,6 @@
         force: yes
   when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-mongo-secondary', 'db-mongo-analytics']
 
-
 - name: Block for mongo volume on hetzner
   block:
     - name: Create backup directory
@@ -119,7 +117,7 @@
     # network_mode: default
     # networks:
     #   - name: newsblurnet
-    #     aliases: 
+    #     aliases:
     #       - mongo
     # ports:
     #   - "27017:27017"
@@ -148,7 +146,7 @@
         # network_mode: default
         # networks:
         #   - name: newsblurnet
-        #     aliases: 
+        #     aliases:
         #       - mongo
         # ports:
         #   - "27017:27017"
@@ -186,7 +184,7 @@
     network_mode: default
     networks:
       - name: newsblurnet
-        aliases: 
+        aliases:
           - mongo
     ports:
       - "27017:27017"
@@ -214,7 +212,7 @@
     network_mode: default
     networks:
       - name: newsblurnet
-        aliases: 
+        aliases:
           - mongo
     ports:
       - "27017:27017"
@@ -231,7 +229,7 @@
 - name: Create mongo database user
   shell:
     # Don't use this line below as it means there is already a username and password, so no need to set one
-    # sleep 2; docker exec mongo mongo -u "{{ mongodb_username }}" -p "{{ mongodb_password }}" --eval ' 
+    # sleep 2; docker exec mongo mongo -u "{{ mongodb_username }}" -p "{{ mongodb_password }}" --eval '
     cmd: >-
       sleep 2; docker exec mongo mongo --eval '
       db.createUser(
@@ -252,8 +250,9 @@
     - "'there are no users authenticated' not in auth_result.stdout"
   tags:
     - mongoauth
+    - never
 
-# - debug: 
+# - debug:
 #     msg: "{{ auth_result }}"
 #   tags:
 #     - mongoauth
@@ -281,9 +280,9 @@
 - name: Setup logrotate for mongo
   become: yes
   copy: src=logrotate.conf dest=/etc/logrotate.d/mongodb mode=0755
-  tags: 
+  tags:
     - logrotate
-  
+
 - name: Add sanity checkers cronjob for disk usage
   become: yes
   cron:
@@ -326,7 +325,6 @@
   tags:
     - mongo-backup
     - cron
-
 # - name: Add mongo starred_stories+stories backup
 #   cron:
 #     name: mongo starred/shared/all stories backup
@@ -338,7 +336,7 @@
 #     - mongo-backup
 
 # Renaming a db-mongo-primary3 to db-mongo-primary2:
-#  - Change hostname to db-mongo-primary2 on Digital Ocean 
+#  - Change hostname to db-mongo-primary2 on Digital Ocean
 #  -     make list; doctl compute droplet-action rename <id>  --droplet-name db-mongo-primary2
 #  - Change hostname to db-mongo-primary2 in /etc/hostname
 #  - make inventory
@@ -352,7 +350,7 @@
 #  - doctl compute droplet delete db-mongo3
 #  - tf state rm "digitalocean_droplet.db-mongo-primary-s[1]"
 #  - tf state rm "digitalocean_droplet.db-mongo-primary-s[2]"
-#  - tf state mv "digitalocean_droplet.db-mongo-primary-s[3]" "digitalocean_droplet.db-mongo-primary-s[1]" 
+#  - tf state mv "digitalocean_droplet.db-mongo-primary-s[3]" "digitalocean_droplet.db-mongo-primary-s[1]"
 #  - Change hostname to db-mongo2 in /etc/hostname
 #  - sudo hostname db-mongo-primary2
 

ansible/roles/postgres-exporter/tasks/get_credentials.py

@@ -1,12 +1,13 @@
 #!/srv/newsblur/venv/newsblur3/bin/python
 import sys
-sys.path.append('/srv/newsblur')
+
+sys.path.append("/srv/newsblur")
 from newsblur_web import settings
 
-username = settings.DATABASES['default']['USER']
-password = settings.DATABASES['default']['PASSWORD']
+username = settings.DATABASES["default"]["USER"]
+password = settings.DATABASES["default"]["PASSWORD"]
 
-if sys.argv[1] =='postgres_credentials':
+if sys.argv[1] == "postgres_credentials":
     print(f"{username}:{password}")
-if sys.argv[1] =='s3_bucket':
-    print(settings.S3_BACKUP_BUCKET)
+if sys.argv[1] == "s3_bucket":
+    print(settings.S3_BACKUP_BUCKET)

ansible/roles/postgres-exporter/tasks/main.yml

@@ -1,24 +1,24 @@
-
 - name: Register Postgres user and password
   become: no
   run_once: yes
   register: postgres_credentials
   local_action: command /srv/newsblur/ansible/roles/postgres-exporter/tasks/get_credentials.py postgres_credentials
+
 - name: Start postgres-exporter container
   become: yes
   docker_container:
     name: postgres-exporter
-    image:  prometheuscommunity/postgres-exporter
+    image: prometheuscommunity/postgres-exporter
     restart_policy: unless-stopped
     container_default_behavior: no_defaults
     networks_cli_compatible: yes
     network_mode: default
     networks:
       - name: newsblurnet
     env:
-      DATA_SOURCE_NAME: 'postgresql://{{ postgres_credentials.stdout }}@db-postgres.service.nyc1.consul:5432/postgres?sslmode=disable'
+      DATA_SOURCE_NAME: "postgresql://{{ postgres_credentials.stdout }}@db-postgres.service.nyc1.consul:5432/postgres?sslmode=disable"
     ports:
-    - '9187:9187'
+      - "9187:9187"
 
 - name: Register postgres-exporter in consul
   tags: consul

ansible/roles/ufw/templates/ufw_rules.sh.j2

@@ -41,3 +41,7 @@ apply_rule "route allow from {{ host }}" "FWD" "{{ host }}"
 apply_rule "allow from {{ host }}" "IN" "{{ host }}"
 apply_rule "route allow from {{ host }}" "FWD" "{{ host }}"
 {% endfor %}
+
+# Allow traffic on docker0 interface
+apply_rule "allow in on docker0" "IN" "docker0"
+apply_rule "allow out on docker0" "IN" "docker0"

ansible/utils/check_droplet.py

@@ -1,7 +1,9 @@
+import subprocess
 import sys
 import time
+
 import digitalocean
-import subprocess
+
 
 def test_ssh(drop):
     droplet_ip_address = drop.ip_address
@@ -10,6 +12,7 @@ def test_ssh(drop):
         return True
     return False
 
+
 TOKEN_FILE = "/srv/secrets-newsblur/keys/digital_ocean.token"
 droplet_name = sys.argv[1]
 
@@ -25,12 +28,12 @@ def test_ssh(drop):
 while not ssh_works:
     if timer > timeout:
         raise Exception(f"The {droplet_name} droplet was not created.")
-    
+
     droplets = [drop for drop in manager.get_all_droplets() if drop.name == droplet_name]
     if droplets:
         droplet = droplets[0]
         print(f"Found the {droplet_name} droplet. IP address is {droplet.ip_address}. Testing ssh...")
         ssh_works = test_ssh(droplet)
     time.sleep(3)
     timer += 3
-print("Success!")
+print("Success!")

ansible/utils/generate_inventory.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python
 
 import os
-import time
-import sys
 import subprocess
+import sys
+import time
+
 import digitalocean
 
 OLD = False
 # Set env var OLD=1 to use existing servers
-if os.environ.get('OLD', False):
+if os.environ.get("OLD", False):
     OLD = True
 
 if OLD:
@@ -17,37 +18,37 @@
     TOKEN_FILE = "/srv/secrets-newsblur/keys/digital_ocean.token"
 
 try:
-    api_token = open(TOKEN_FILE, 'r').read().strip()
+    api_token = open(TOKEN_FILE, "r").read().strip()
 except IOError:
     print(f" ---> Missing Digital Ocean API token: {TOKEN_FILE}")
     exit()
 
 outfile = f"/srv/newsblur/ansible/inventories/digital_ocean{'.old' if OLD else ''}.ini"
 
 # Install from https://github.com/do-community/do-ansible-inventory/releases
-ansible_inventory_cmd = f'do-ansible-inventory -t {api_token} --out {outfile}'
+ansible_inventory_cmd = f"do-ansible-inventory -t {api_token} --out {outfile}"
 subprocess.call(ansible_inventory_cmd, shell=True)
 
-with open(outfile, 'r') as original: 
+with open(outfile, "r") as original:
     data = original.read()
-with open(outfile, 'w') as modified: 
+with open(outfile, "w") as modified:
     modified.write("127.0.0.1 ansible_connection=local\n" + data)
 
-exit() # Too many requests if we run the below code
+exit()  # Too many requests if we run the below code
 
 do = digitalocean.Manager(token=api_token)
 droplets = do.get_all_droplets()
 
-print("\n ---> Checking droplets: %s\n" % (' '.join([d.name for d in droplets])))
+print("\n ---> Checking droplets: %s\n" % (" ".join([d.name for d in droplets])))
 
 
 def check_droplets_created():
     i = 0
     droplets = do.get_all_droplets()
 
     for instance in droplets:
-        if instance.status == 'new':
-            print(".", end=' ')
+        if instance.status == "new":
+            print(".", end=" ")
             sys.stdout.flush()
             i += 1
             time.sleep(i)
@@ -56,6 +57,7 @@ def check_droplets_created():
         print(" ---> All booted!")
         return True
 
+
 i = 0
 while True:
     if check_droplets_created():