Initial commit

README.md

@@ -1,2 +1,55 @@
 # linky
-Script collecting data from a Linky (French power meter) using serial output
+
+(This README is in French only because this subject is specific to France)
+
+Ce dépôt Github contient un script Python permettant de lire les données en provenance du bornier "Téléinformation" d'un compteur Linky.
+
+Pour plus d'informations sur comment brancher et configurer la liaison série, lisez ces articles :
+- [Partie 1](https://notes.ailothaen.fr/post/2022/07/Mesures-et-graphiques-de-la-consommation-d-un-compteur-Linky-avec-un-Raspberry-Pi-et-Grafana-%E2%80%93-Partie-1/2-%28mat%C3%A9riel%29)
+- [Partie 2](https://notes.ailothaen.fr/post/2022/07/Mesures-et-graphiques-d-un-compteur-Linky-avec-un-Raspberry-Pi-et-Grafana-%E2%80%93-Partie-2/2-%28logiciel%29)
+
+## Dépendances
+
+Pour fonctionner, ce script requiert une base de données type MySQL, ainsi que quelques dépendances Python.
+
+Voici les commandes classiques sur MariaDB pour créer une base de données, créer un utilisateur, et donner tous les droits à cet utilisateur sur cette base de données :
+
+```
+mysql> CREATE DATABASE linky;
+mysql> CREATE USER 'linky'@'localhost' IDENTIFIED BY 'motdepasse';
+mysql> GRANT ALL PRIVILEGES ON linky.* TO 'linky'@'localhost';
+mysql> FLUSH PRIVILEGES;
+```
+
+Installez les dépendances Python avec `python3 -m pip install -r requirements.txt`
+
+
+## Mise en place du service
+
+Mettez le fichier `linky.service` dans `/etc/systemd/system` (sur Debian ; l'emplacement est peut-être différent selon la distribution).  
+Éditez le fichier selon l'endroit où vous mettrez les scripts, et l'utilisateur que vous utiliserez.
+
+
+## Démarrage
+
+```
+systemctl daemon-reload
+systemctl enable linky
+systemctl start linky
+```
+
+Si vous avez bien tout installé, vous devriez commencer à voir des lignes (une ligne par minute) dans la table "stream".
+
+```
+MariaDB [linky]> select * from stream;
++---------------------+---------+------+-----------+
+| clock               | BASE    | PAPP | BASE_diff |
++---------------------+---------+------+-----------+
+| 2022-07-14 17:02:55 | 2086442 |   70 |         0 |
+| 2022-07-14 17:03:57 | 2086443 |   70 |         1 |
+| 2022-07-14 17:04:58 | 2086443 |   70 |         0 |
++---------------------+---------+------+-----------+
+4 rows in set (0.002 sec)
+```
+
+Sinon, regardez les logs du script (`logs`) ou du service (`journalctl -u linky`)

config.yml

@@ -0,0 +1,7 @@
+database:
+  server: 127.0.0.1
+  name: linky
+  user: linky
+  password: mypassword
+device:
+  file: /dev/ttyS0

linky.py

@@ -0,0 +1,167 @@
+#!/usr/bin/python3
+
+# stdlib
+import serial, MySQLdb, datetime, sys, logging, logging.handlers
+
+# 3rd party
+import yaml
+
+
+def init_log_system():
+    """
+    Initializes log system
+    """
+    log = logging.getLogger('linky')
+    log.setLevel(logging.DEBUG) # Define minimum severity here
+    handler = logging.handlers.RotatingFileHandler('./logs/linky.log', maxBytes=1000000, backupCount=5) # Log file of 1 MB, 5 previous files kept
+    formatter = logging.Formatter('[%(asctime)s][%(module)s][%(levelname)s] %(message)s', '%Y-%m-%d %H:%M:%S %z') # Custom line format and time format to include the module and delimit all of this well
+    handler.setFormatter(formatter)
+    log.addHandler(handler)
+    return log
+
+
+def load_config():
+    """
+    Loads config file
+    """
+    try:
+        with open('config.yml', 'r') as f:
+            config = yaml.safe_load(f)
+    except Exception as e:
+        log.critical('Something went wrong while opening config file:', exc_info=True)
+        print('Something went wrong while opening config file. See logs for more info.', file=sys.stderr)
+        raise SystemExit(3)
+    else:
+        return config
+
+
+def setup_serial(dev):
+    """
+    Builds the serial connection object.
+
+    Args:
+        dev (str): Linux device of the connector (like "/dev/ttyS0")
+    """
+    terminal = serial.Serial()
+    terminal.port = dev
+    terminal.baudrate = 1200
+    terminal.stopbits = serial.STOPBITS_ONE
+    terminal.bytesize = serial.SEVENBITS
+    return terminal
+
+
+def test_db_connection(server, user, password, name):
+    """
+    Tests DB connection, and also creates the schema if missing
+
+    Args:
+        server (str): Database server
+        user (str): Database user
+        password (str): Database user password
+        name (str): Database name
+    """
+    # testing connection
+    db, cr = open_db(server, user, password, name)
+
+    # create schema if first connection
+    stream_exists = cr.execute(f"SELECT * FROM information_schema.tables WHERE table_schema = '{name}' AND table_name = 'stream' LIMIT 1;")
+    dailies_exists = cr.execute(f"SELECT * FROM information_schema.tables WHERE table_schema = '{name}' AND table_name = 'dailies' LIMIT 1;")
+
+    if stream_exists == 0 or dailies_exists == 0:
+        log.info("Database schema is not there, creating it...")
+        try:
+            cr.execute("CREATE TABLE `dailies` (`clock` date NOT NULL, `BASE_diff` int(11) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")
+            cr.execute("CREATE TABLE `stream` (`clock` datetime NOT NULL, `BASE` int(11) NOT NULL, `PAPP` int(11) NOT NULL, `BASE_diff` int(11) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")
+        except MySQLdb._exceptions.OperationalError:
+            log.critical('Something went wrong while trying to create database schema:', exc_info=True)
+            print('Something went wrong while trying to create database schema. See logs for more info.', file=sys.stderr)
+            raise SystemExit(4)
+        else:
+            log.info("Database schema created successfully")
+
+
+def open_db(server, user, password, name):
+    """
+    Connects to database
+
+    Args:
+        server (str): Database server
+        user (str): Database user
+        password (str): Database user password
+        name (str): Database name
+    """
+    try:
+        db = MySQLdb.connect(server, user, password, name)
+        cr = db.cursor()
+        return db, cr
+    except MySQLdb._exceptions.OperationalError:
+        log.critical('Something went wrong while connecting to database server:', exc_info=True)
+        print('Something went wrong while connecting to database server. See logs for more info.', file=sys.stderr)
+        raise SystemExit(4)
+
+
+def close_db(db):
+    """
+    Closes connection to database
+
+    Args:
+        db (type): MySQLdb database object
+    """
+    db.close()
+
+
+def insert_stream(db, cr, BASE, PAPP):
+    """
+    Insert a record in the stream table
+
+    Args:
+        db (type): MySQLdb database object
+        cr (type): MySQLdb cursor object
+        BASE (int): Linky BASE value (Wh meter)
+        PAPP (int): Linky PAPP value (current VA power)
+    """
+    # generating time
+    now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
+
+    # retrieving previous BASE and calculating BASE_diff
+    cr.execute("SELECT BASE FROM stream ORDER BY clock DESC LIMIT 1;")
+    try:
+        previous = cr.fetchone()[0]
+    except TypeError:
+        # no records yet
+        BASE_diff = 0
+    else:
+        BASE_diff = BASE-int(previous)
+
+    # inserting records
+    cr.execute(f"INSERT INTO stream VALUES ('{now}', {BASE}, {PAPP}, {BASE_diff});")
+    db.commit()
+
+
+def insert_dailies(db, cr, BASE):
+    """
+    Inserts a record in the dailies table
+
+    Args:
+        db (type): MySQLdb database object
+        cr (type): MySQLdb cursor object
+        BASE (int): Linky BASE value (Wh meter)
+    """
+    # getting previous day midnight BASE value
+    cr.execute("SELECT clock, BASE from `stream` INNER JOIN (SELECT MIN(clock) AS firstOfTheDay FROM `stream` GROUP BY DATE(clock)) joint ON `stream`.clock = joint.firstOfTheDay ORDER BY `stream`.clock DESC LIMIT 1;")
+    try:
+        previous = cr.fetchone()[1]
+    except TypeError:
+        # no records yet
+        diff = 0
+    else:
+        diff = BASE-previous
+
+    now = datetime.datetime.utcnow().strftime('%Y-%m-%d')
+
+    cr.execute(f'INSERT INTO dailies VALUES ("{now}", "{diff}")')
+    db.commit()
+
+
+# Initializing log system
+log = init_log_system()

linky.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Python service fetching Linky and putting data into MySQL
+After=network-online.target
+After=mariadb.service
+
+[Service]
+Type=simple
+User=root
+Group=root
+ExecStart=/usr/bin/python3 /srv/linky/main.py
+WorkingDirectory=/srv/linky/
+
+[Install]
+WantedBy=multi-user.target

main.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+
+# stdlib
+import datetime, time, logging
+
+# Self libraries
+import linky
+
+
+# ----------------------------- #
+# Setup                         #
+# ----------------------------- #
+
+log = logging.getLogger('linky')
+
+log.debug('Loading config...')
+config = linky.load_config()
+log.debug(f'Config loaded! Values: {config}')
+
+terminal = linky.setup_serial(config['device']['file'])
+
+# Trying to connect to db server and creating schema if not exists
+linky.test_db_connection(config['database']['server'], config['database']['user'], config['database']['password'], config['database']['name'])
+
+
+# ----------------------------- #
+# Main loop                     #
+# ----------------------------- #
+
+current_loop_day = datetime.datetime.now(datetime.timezone.utc).day
+previous_loop_day = datetime.datetime.now(datetime.timezone.utc).day
+
+while True:
+    log.debug("Cycle begins")
+    data_BASE = None
+    data_PAPP = None
+    current_loop_day = datetime.datetime.now(datetime.timezone.utc).day
+
+    # Now beginning to read data from Linky
+    log.debug("Opening terminal...")
+    terminal.open()
+
+    # reading continously output until we have data that interests us
+    while True:
+        line = terminal.readline().decode('ascii')
+        log.debug(f"Current line: {line}")
+
+        if line.startswith('BASE'):
+            data_BASE = int(line.split(' ')[1])
+        if line.startswith('PAPP'):
+            data_PAPP = int(line.split(' ')[1])
+
+        # We have BASE and PAPP, we can now close the connection
+        if data_BASE and data_PAPP:
+            log.debug(f"Output parsed: BASE={data_BASE}, PAPP={data_PAPP}. Closing terminal.")
+            terminal.close()
+            break
+
+    # Connecting to database
+    log.debug("Connecting to database")
+    db, cr = linky.open_db(config['database']['server'], config['database']['user'], config['database']['password'], config['database']['name'])
+
+    # first record of the day? generating dailies
+    if current_loop_day != previous_loop_day:
+        log.debug(f"First record of the day! Inserting dailies record.")
+        linky.insert_dailies(db, cr, data_BASE)
+    previous_loop_day = datetime.datetime.utcnow().day
+
+    # inserting values
+    log.debug("Inserting stream record")
+    linky.insert_stream(db, cr, data_BASE, data_PAPP)
+
+    log.debug("Cycle ends, sleeping for 60 seconds")
+    time.sleep(60)

requirements.txt

@@ -0,0 +1,5 @@
+# Automatically generated by https://github.com/damnever/pigar.
+
+PyYAML >= 5.4.1
+mysqlclient >= 2.0.3
+pyserial >= 3.5