[O2B-1645] Distinguish the type of API clients in the API logs

docs/backend-architecture.md

@@ -41,8 +41,8 @@ Currently, `usecases` returns `HTTP` oriented objects when an exception occurs,
 Every request handled by the `HTTP` API (`/api`, not the static files) and every `gRPC` call is logged once handled, by `ApiRequestLogger` (`HTTP`: `accessLogMiddleware`, `gRPC`: `bindGRPCController`). The entry is a single-line `JSON`, logged by the `API` logger **to the local log only** (it is not forwarded to InfoLogger). `HTTP` and `gRPC` entries share the same structure, so that they can be stored and aggregated together in OpenSearch:
 
 ```json
-{"type":"api_request","protocol":"http","method":"GET","route":"/api/runs","status":200,"durationMs":42,"userId":7,"bytes":18234,"params":"{\"filter\":{\"runNumbers\":\"1,2\"},\"page\":{\"limit\":\"50\"}}"}
-{"type":"api_request","protocol":"grpc","method":null,"route":"/o2.bookkeeping.RunService/Get","status":0,"durationMs":12,"userId":null,"bytes":null,"params":"{\"runNumber\":12}"}
+{"type":"api_request","protocol":"http","method":"GET","route":"/api/runs","status":200,"durationMs":42,"userId":7,"bytes":18234,"client":"browser","device":"desktop","params":"{\"filter\":{\"runNumbers\":\"1,2\"},\"page\":{\"limit\":\"50\"}}"}
+{"type":"api_request","protocol":"grpc","method":null,"route":"/o2.bookkeeping.RunService/Get","status":0,"durationMs":12,"userId":null,"bytes":null,"client":"grpc-node","device":null,"params":"{\"runNumber\":12}"}
 ```
 
 - `type`: always `api_request`
@@ -53,7 +53,9 @@ Every request handled by the `HTTP` API (`/api`, not the static files) and every
 - `durationMs`: time between the reception of the request by the middleware (`HTTP`) or the call (`gRPC`) and the end of the response, so it includes the authentication
 - `userId`: the id of the user, `null` if not authenticated
 - `bytes`: the `content-length` of the `HTTP` response, `null` if not provided and for `gRPC`
-- `params`: the `HTTP` query string (the `token` parameter is removed and the body is never logged) or the `gRPC` request message, as a `JSON` **string** (not an object, because the structure varies from one endpoint to another and the types are not the same between protocols, which would lead to conflicts in OpenSearch mappings). It is omitted when empty and truncated (with a trailing `…`) when it exceeds 1000 characters
+- `client`: the kind of client, identified from the `User-Agent` it declares (`browser`, `curl`, `wget`, `python`, `node`, `go`, `java`, `postman`, `grpc-node`, `grpc-cxx`, `grpc-python`, `grpc-go`, `grpc-java`, `grpc-other` or `unknown`). It is self-reported, so it is meant for statistics only and must never be used for security purposes. The proxy in front of the application (for example nginx) must forward the `User-Agent` header, which is the default behavior
+- `device`: for browsers only (`null` otherwise), `phone`, `tablet` or `desktop`, deduced from the `User-Agent` and, for chromium based browsers, the `Sec-CH-UA-Mobile` header. An iPad requesting the desktop version of websites (default since iPadOS 13) can not be told apart from a Mac
+- `params`: the `HTTP` query string (the `token` parameter is removed and the body is never logged) or the `gRPC` request message, as a `JSON` **string** (not an object, because the structure varies from one endpoint to another and the types are not the same between protocols, which would lead to conflicts in OpenSearch mappings). It is omitted when empty and truncated (with a trailing `…`) when it exceeds 500 characters
 - `aborted`: `true` only if the connection was closed before the response was completely sent (`HTTP` only)
 
 Nothing is logged in test mode.

lib/domain/enums/ApiClients.js

@@ -0,0 +1,33 @@
+/**
+ *  @license
+ *  Copyright CERN and copyright holders of ALICE O2. This software is
+ *  distributed under the terms of the GNU General Public License v3 (GPL
+ *  Version 3), copied verbatim in the file "COPYING".
+ *
+ *  See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ *  In applying this license CERN does not waive the privileges and immunities
+ *  granted to it by virtue of its status as an Intergovernmental Organization
+ *  or submit itself to any jurisdiction.
+ */
+
+/**
+ * Types of clients of the API, as identified from the user agent the request contains
+ */
+exports.ApiClients = Object.freeze({
+    BROWSER: 'browser',
+    CURL: 'curl',
+    WGET: 'wget',
+    PYTHON: 'python',
+    NODE: 'node',
+    GO: 'go',
+    JAVA: 'java',
+    POSTMAN: 'postman',
+    GRPC_NODE: 'grpc-node',
+    GRPC_CXX: 'grpc-cxx',
+    GRPC_PYTHON: 'grpc-python',
+    GRPC_GO: 'grpc-go',
+    GRPC_JAVA: 'grpc-java',
+    GRPC_OTHER: 'grpc-other',
+    UNKNOWN: 'unknown',
+});

lib/domain/enums/ApiDevices.js

@@ -0,0 +1,21 @@
+/**
+ *  @license
+ *  Copyright CERN and copyright holders of ALICE O2. This software is
+ *  distributed under the terms of the GNU General Public License v3 (GPL
+ *  Version 3), copied verbatim in the file "COPYING".
+ *
+ *  See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ *  In applying this license CERN does not waive the privileges and immunities
+ *  granted to it by virtue of its status as an Intergovernmental Organization
+ *  or submit itself to any jurisdiction.
+ */
+
+/**
+ * Types of devices from which the API is used by a browser
+ */
+exports.ApiDevices = Object.freeze({
+    PHONE: 'phone',
+    TABLET: 'tablet',
+    DESKTOP: 'desktop',
+});

lib/server/gRPC/bindGRPCController.js

@@ -181,6 +181,7 @@ const bindGRPCController = (
                     status,
                     durationMs: stopTimer(),
                     userId: call.identity?.id ?? null,
+                    userAgent: call.metadata?.get('user-agent')?.[0],
                     // The controller has not been reached (for example, a pre-processor failed): the request is still untouched
                     params: requestSnapshot === undefined ? call.request : requestSnapshot,
                 });

lib/server/middleware/accessLog.middleware.js

@@ -59,6 +59,8 @@ exports.accessLogMiddleware = (configuration) => {
                 durationMs: stopTimer(),
                 userId: session?.id ?? null,
                 bytes: Number.isNaN(contentLength) ? null : contentLength,
+                userAgent: request.headers?.['user-agent'],
+                mobileHint: request.headers?.['sec-ch-ua-mobile'],
                 params,
                 aborted: !response.writableFinished,
             });

lib/server/utilities/ApiRequestLogger.js

@@ -13,6 +13,7 @@
 
 const { LogManager } = require('@aliceo2/web-ui');
 const { isInTestMode } = require('../../utilities/env-utils');
+const { identifyClient } = require('./identifyClient.js');
 
 const NANOSECONDS_PER_MILLISECOND = 1000000n;
 
@@ -69,18 +70,21 @@ class ApiRequestLogger {
      * @param {number} request.durationMs the duration of the request
      * @param {number|null} [request.userId] the id of the user
      * @param {number|null} [request.bytes] the size of the response
+     * @param {string} [request.userAgent] the user agent declared by the client, used to identify its kind and device
+     * @param {string} [request.mobileHint] the value of the Sec-CH-UA-Mobile header
      * @param {object|string|null} [request.params] the HTTP query or gRPC request message (or its already stringified snapshot), logged as a
      *     (truncated) JSON string, omitted if empty
      * @param {boolean} [request.aborted] true if the request has been interrupted before the end of the response
      * @return {void}
      */
-    log({ protocol, method = null, route, status, durationMs, userId = null, bytes = null, params, aborted }) {
+    log({ protocol, method = null, route, status, durationMs, userId = null, bytes = null, userAgent, mobileHint, params, aborted }) {
         if (this._silent) {
             return;
         }
 
         try {
-            const entry = { type: 'api_request', protocol, method, route, status, durationMs, userId, bytes };
+            const { client, device } = identifyClient({ userAgent, mobileHint });
+            const entry = { type: 'api_request', protocol, method, route, status, durationMs, userId, bytes, client, device };
 
             const stringifiedParams = ApiRequestLogger.stringifyParams(params);
             if (stringifiedParams !== null) {

lib/server/utilities/identifyClient.js

@@ -0,0 +1,81 @@
+/**
+ *  @license
+ *  Copyright CERN and copyright holders of ALICE O2. This software is
+ *  distributed under the terms of the GNU General Public License v3 (GPL
+ *  Version 3), copied verbatim in the file "COPYING".
+ *
+ *  See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ *  In applying this license CERN does not waive the privileges and immunities
+ *  granted to it by virtue of its status as an Intergovernmental Organization
+ *  or submit itself to any jurisdiction.
+ */
+
+const { ApiClients } = require('../../domain/enums/ApiClients.js');
+const { ApiDevices } = require('../../domain/enums/ApiDevices.js');
+
+/**
+ * Ordered list of the clients patterns: Postman, the gRPC clients, then the specific tools must come before the generic browser pattern
+ *
+ * @type {Array<[RegExp, ApiClients]>}
+ */
+const CLIENTS_PATTERNS = [
+    // Postman must come first: its HTTP user agent is PostmanRuntime/x, its gRPC one is the Node gRPC one suffixed (grpc-node-js/x-postman.y)
+    [/postman/i, ApiClients.POSTMAN],
+    // The gRPC clients must come first, as their user agent may contain the name of the underlying library (for example grpc-java-okhttp)
+    [/grpc-node/i, ApiClients.GRPC_NODE],
+    [/grpc-c\+\+/i, ApiClients.GRPC_CXX],
+    [/grpc-python/i, ApiClients.GRPC_PYTHON],
+    [/grpc-go/i, ApiClients.GRPC_GO],
+    [/grpc-java/i, ApiClients.GRPC_JAVA],
+    [/grpc-/i, ApiClients.GRPC_OTHER],
+    [/^curl\//i, ApiClients.CURL],
+    [/^wget\//i, ApiClients.WGET],
+    [/python-requests|python-urllib|python-httpx|aiohttp|^python\//i, ApiClients.PYTHON],
+    [/axios|node-fetch|undici|^node$/i, ApiClients.NODE],
+    [/go-http-client/i, ApiClients.GO],
+    // JVM clients: HttpURLConnection (Java/x), the JDK HttpClient, Apache HttpClient and OkHttp (also used by Android and Kotlin applications)
+    [/^Java\/|^Java-http-client|Apache-HttpClient|okhttp/i, ApiClients.JAVA],
+    [/^Mozilla\/5\.0.*(chrome|crios|safari|firefox|fxios|edg|opr)\//i, ApiClients.BROWSER],
+];
+
+/**
+ * Identify the device of a browser
+ *
+ * @param {string} userAgent the user agent of the browser
+ * @param {string} [mobileHint] the value of the Sec-CH-UA-Mobile header (sent by chromium based browsers: ?1 for phones, ?0 otherwise)
+ * @return {ApiDevices} the device
+ */
+const identifyDevice = (userAgent, mobileHint) => {
+    if (/iPad|Tablet/i.test(userAgent)) {
+        return ApiDevices.TABLET;
+    }
+    if (/iPhone|iPod|Windows Phone|Opera Mini/i.test(userAgent)) {
+        return ApiDevices.PHONE;
+    }
+    // Android tablets do not declare "Mobile" in their user agent, phones do
+    if (/Android/i.test(userAgent)) {
+        return /Mobile/i.test(userAgent) ? ApiDevices.PHONE : ApiDevices.TABLET;
+    }
+    return /Mobile/i.test(userAgent) || mobileHint === '?1' ? ApiDevices.PHONE : ApiDevices.DESKTOP;
+};
+
+/**
+ * Identify the client of the API from the user agent it declares. This is self-reported, so it is meant for statistics only and must never be
+ * used for security purposes. Note that an iPad requesting the desktop version of websites (default since iPadOS 13) can not be told apart from
+ * a Mac
+ *
+ * @param {object} [headers] the user agent and optional client hints
+ * @param {string} [headers.userAgent] the user agent
+ * @param {string} [headers.mobileHint] the value of the Sec-CH-UA-Mobile header
+ * @return {{client: ApiClients, device: (ApiDevices|null)}} the client and, for browsers only, the device (null otherwise)
+ */
+exports.identifyClient = ({ userAgent, mobileHint } = {}) => {
+    if (typeof userAgent !== 'string') {
+        return { client: ApiClients.UNKNOWN, device: null };
+    }
+
+    const [, client = ApiClients.UNKNOWN] = CLIENTS_PATTERNS.find(([pattern]) => pattern.test(userAgent)) ?? [];
+
+    return { client, device: client === ApiClients.BROWSER ? identifyDevice(userAgent, mobileHint) : null };
+};

test/gRPC/serviceImplementation.test.js

@@ -169,7 +169,14 @@ module.exports = () => {
         it('should log successful calls with the user, the method and the request', async () => {
             const { adapter, logger } = getAdapter({ TestEnums: () => ({}), TestBigInts: () => ({ ui: 1n, i: 2n }), TestRepeated: () => ({}) });
 
-            await adapter.TestBigInts({ identity: { id: 9 }, request: { ui: Long.fromString('12', true), i: Long.fromString('-5') } }, sinon.fake());
+            await adapter.TestBigInts(
+                {
+                    identity: { id: 9 },
+                    metadata: { get: (name) => name === 'user-agent' ? ['grpc-c++/1.60.0 grpc-c/37.0.0'] : [] },
+                    request: { ui: Long.fromString('12', true), i: Long.fromString('-5') },
+                },
+                sinon.fake(),
+            );
 
             expect(logger.infoMessage.calledOnce).to.be.true;
             const { durationMs, ...entry } = JSON.parse(logger.infoMessage.firstCall.args[0]);
@@ -182,6 +189,8 @@ module.exports = () => {
                 status: 0,
                 userId: 9,
                 bytes: null,
+                client: 'grpc-cxx',
+                device: null,
                 params: '{"ui":"12","i":"-5"}',
             });
         });

test/lib/server/middleware/accessLog.middleware.test.js

@@ -41,7 +41,15 @@ module.exports = () => {
     it('should log a single line JSON entry with route pattern, status, duration, user, size and params once the response is sent', () => {
         const query = { filter: { runNumbers: '1,2' }, page: { limit: '10' } };
         const { logger, next, response } = handle(
-            { method: 'GET', baseUrl: '/api', originalUrl: '/api/runs/12?a=1', route: { path: '/runs/:runNumber' }, session: { id: 7 }, query },
+            {
+                method: 'GET',
+                baseUrl: '/api',
+                originalUrl: '/api/runs/12?a=1',
+                route: { path: '/runs/:runNumber' },
+                session: { id: 7 },
+                query,
+                headers: { 'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 Safari/604.1' },
+            },
             { headers: { 'content-length': '123' }, statusCode: 201 },
         );
 
@@ -65,16 +73,40 @@ module.exports = () => {
             status: 201,
             userId: 7,
             bytes: 123,
+            client: 'browser',
+            device: 'phone',
             params: JSON.stringify(query),
         });
     });
 
+    it('should identify the client from the user agent and the mobile hint', () => {
+        const { logger, response } = handle({
+            method: 'GET',
+            baseUrl: '/api',
+            originalUrl: '/api/runs',
+            route: { path: '/runs' },
+            headers: { 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) Chrome/120.0 Safari/537.36', 'sec-ch-ua-mobile': '?1' },
+        });
+        response.emit('finish');
+        expect(JSON.parse(logger.infoMessage.firstCall.args[0])).to.include({ client: 'browser', device: 'phone' });
+
+        const { logger: curlLogger, response: curlResponse } = handle({
+            method: 'GET',
+            baseUrl: '/api',
+            originalUrl: '/api/runs',
+            route: { path: '/runs' },
+            headers: { 'user-agent': 'curl/8.4.0' },
+        });
+        curlResponse.emit('finish');
+        expect(JSON.parse(curlLogger.infoMessage.firstCall.args[0])).to.include({ client: 'curl', device: null });
+    });
+
     it('should log null user and size, no params, and the path when the request is anonymous and no route matched', () => {
         const { logger, response } = handle({ method: 'GET', baseUrl: '/api', originalUrl: '/api/unknown?x=1', query: {} });
         response.emit('finish');
 
         const entry = JSON.parse(logger.infoMessage.firstCall.args[0]);
-        expect(entry).to.include({ route: '/api/unknown', userId: null, bytes: null });
+        expect(entry).to.include({ route: '/api/unknown', userId: null, bytes: null, client: 'unknown', device: null });
         expect(entry).to.not.have.property('params');
         expect(entry).to.not.have.property('aborted');
     });

test/lib/server/utilities/ApiRequestLogger.test.js

@@ -22,7 +22,7 @@ module.exports = () => {
         const logger = { infoMessage: sinon.fake() };
         const apiRequestLogger = new ApiRequestLogger({ logger, silent: false });
 
-        apiRequestLogger.log({ ...request, userId: 4, params: { runNumber: 12 } });
+        apiRequestLogger.log({ ...request, userId: 4, userAgent: 'grpc-node-js/1.9.0', params: { runNumber: 12 } });
         apiRequestLogger.log({ protocol: 'http', method: 'GET', route: '/api/runs', status: 200, durationMs: 5, bytes: 10 });
 
         expect(logger.infoMessage.callCount).to.equal(2);
@@ -40,6 +40,8 @@ module.exports = () => {
             durationMs: 3,
             userId: 4,
             bytes: null,
+            client: 'grpc-node',
+            device: null,
             params: '{"runNumber":12}',
         });
         expect(Object.keys(httpEntry)).to.eql(Object.keys(grpcEntry).filter((key) => key !== 'params'));