doc/examples: document usage of sql database

Update the doc and examples to use the sql database instead of
etcd/datamanager.

doc/devel.md

@@ -27,13 +27,7 @@ make
 
 ### Start the agola server
 
-- Copy the `example/config.yml` where you prefer
-
-```
-./bin/agola serve --embedded-etcd --config /path/to/your/config.yml --components all-base,executor
-```
-
-or use an external etcd (set it in the config.yml):
+- Copy the `example/agolademo/config.yml` where you prefer
 
 ```
 ./bin/agola serve --config /path/to/your/config.yml --components all-base,executor

examples/agolademo/config.yml

@@ -24,13 +24,15 @@ notification:
   webExposedURL: "http://172.17.0.1:8000"
   runserviceURL: "http://localhost:4000"
   configstoreURL: "http://localhost:4002"
-  etcd:
-    endpoints: "http://localhost:2379"
+  db:
+    type: sqlite3
+    connString: /data/agola/notification/db
 
 configstore:
   dataDir: /data/agola/configstore
-  etcd:
-    endpoints: "http://localhost:2379"
+  db:
+    type: sqlite3
+    connString: /data/agola/configstore/db
   objectStorage:
     type: posix
     path: /data/agola/configstore/ost
@@ -40,8 +42,9 @@ configstore:
 runservice:
   #debug: true
   dataDir: /data/agola/runservice
-  etcd:
-    endpoints: "http://localhost:2379"
+  db:
+    type: sqlite3
+    connString: /data/agola/runservice/db
   objectStorage:
     type: posix
     path: /data/agola/runservice/ost

examples/kubernetes/distributed/README.md

@@ -4,12 +4,10 @@ This is a distributed deployment where all the components are replicated to achi
 
 Users should use it as an example base setup and change/improve it based on their needs (choosing which object storage to use).
 
-* point to an external etcd cluster
-* points to an external s3 object storage.
-* create 4 deployments for the various components with multiple replicas:
-  * runservice
-  * executor
-  * configstore
-  * gateway / scheduler
-
-
+- point to external postgres databases (one for configstore, one for runservice and one for notification service)
+- points to an external s3 object storage.
+- create 4 deployments for the various components with multiple replicas:
+  - runservice
+  - executor
+  - configstore
+  - gateway / scheduler

examples/kubernetes/distributed/agola.yml

@@ -1,4 +1,3 @@
-
 # The client service. It's a node port for easier testing on minikube. Change
 # it to become a LoadBalancer if needed.
 apiVersion: v1
@@ -15,7 +14,6 @@ spec:
   type: NodePort
 
 ---
-
 # The service for internal components communication with the runservice.
 apiVersion: v1
 kind: Service
@@ -29,7 +27,6 @@ spec:
     component: runservice
 
 ---
-
 # The service for internal components communication with the configstore.
 apiVersion: v1
 kind: Service
@@ -43,7 +40,6 @@ spec:
     component: configstore
 
 ---
-
 # The service for internal components communication with the gitserver.
 apiVersion: v1
 kind: Service
@@ -57,7 +53,6 @@ spec:
     component: gitserver
 
 ---
-
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -94,13 +89,17 @@ data:
       webExposedURL: "http://192.168.39.188:30002"
       runserviceURL: "http://agola-runservice:4000"
       configstoreURL: "http://agola-configstore:4002"
-      etcd:
-        endpoints: "http://localhost:2379"
+      db:
+        # example with a postgres db
+        type: postgres
+        connString: "postgres://@postgres-service/agola_notification?sslmode=disable"
 
     configstore:
       dataDir: /mnt/agola/local/configstore
-      etcd:
-        endpoints: "http://etcd:2379"
+      db:
+        # example with a postgres db
+        type: postgres
+        connString: "postgres://@postgres-service/agola_configstore?sslmode=disable"
       objectStorage:
         type: s3
         # example with minio
@@ -114,8 +113,10 @@ data:
     runservice:
       #debug: true
       dataDir: /mnt/agola/local/runservice
-      etcd:
-        endpoints: "http://etcd:2379"
+      db:
+        # example with a postgres db
+        type: postgres
+        connString: "postgres://@postgres-service/agola_runservice?sslmode=disable"
       objectStorage:
         type: s3
         # example with minio
@@ -144,7 +145,6 @@ data:
         listenAddress: ":4003"
 
 ---
-
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -186,7 +186,6 @@ spec:
         - name: agola-localdata
           emptyDir: {}
 ---
-
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -229,7 +228,6 @@ spec:
           emptyDir: {}
 
 ---
-
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -272,7 +270,6 @@ spec:
           emptyDir: {}
 
 ---
-
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -315,7 +312,6 @@ spec:
           emptyDir: {}
 
 ---
-
 # The gitserver. Since it'll primarily store temporary git build data the
 # simple way to deploy it is to use a deployment with 1 replica and an emptyDir
 # volume. A statefulset with 1 replica and a persistent volume will be a better

examples/kubernetes/simple/README.md

@@ -2,9 +2,8 @@
 
 This is the simplest (and not production ready deployment).
 
-* uses an embedded etcd
-* creates a `PersistentVolumeClaim` that will be used as the object storage container for all the components
-* created a deployment with a single replica
-
-You must not increase the replicas or every pod will uses a different embedded etcd causing many issues and errors (and also the pods will fail if scheduled on different k8s node since the PV for the object storage cannot be mounted on multiple nodes)
+- uses a local sqlite db.
+- creates a `PersistentVolumeClaim` that will be used as the object storage container for all the components
+- created a deployment with a single replica
 
+You MUST NOT increase the replicas or every pod will uses a different sqlite db causing many issues and errors (and also the pods will fail if scheduled on different k8s node since the PV for the object storage cannot be mounted on multiple nodes)

examples/kubernetes/simple/agola.yml

@@ -11,7 +11,6 @@ spec:
   storageClassName: standard
 
 ---
-
 # The client service. It's a node port for easier testing on minikube. Change
 # it to become a LoadBalancer if needed.
 apiVersion: v1
@@ -28,7 +27,6 @@ spec:
   type: NodePort
 
 ---
-
 # The service for internal components communication.
 # We are using an headless service since some k8s deployment doesn't have
 # hairpin mode enabled and pods cannot communicate with themself via a
@@ -52,7 +50,6 @@ spec:
   clusterIP: None
 
 ---
-
 # The agola config
 
 apiVersion: v1
@@ -91,13 +88,15 @@ data:
       webExposedURL: "http://192.168.39.188:30002"
       runserviceURL: "http://agola-internal:4000"
       configstoreURL: "http://agola-internal:4002"
-      etcd:
-        endpoints: "http://localhost:2379"
+      db:
+        type: sqlite3
+        connString: "/opt/agola/notification/db/db.db"
 
     configstore:
       dataDir: /mnt/agola/local/configstore
-      etcd:
-        endpoints: "http://localhost:2379"
+      db:
+        type: sqlite3
+        connString: "/opt/agola/configstore/db/db.db"
       objectStorage:
         type: posix
         path: /mnt/agola/objectstorage/configstore/ost
@@ -107,8 +106,9 @@ data:
     runservice:
       #debug: true
       dataDir: /mnt/agola/local/runservice
-      etcd:
-        endpoints: "http://localhost:2379"
+      db:
+        type: sqlite3
+        connString: "/opt/agola/runservice/db/db.db"
       objectStorage:
         type: posix
         path: /mnt/agola/objectstorage/runservice/ost
@@ -132,15 +132,13 @@ data:
       web:
         listenAddress: ":4003"
 ---
-
-
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: agola
 spec:
   # Do not increase replica count or everything will break since every pod will
-  # have its own etcd instance
+  # have its own database
   replicas: 1
   selector:
     matchLabels:
@@ -156,7 +154,6 @@ spec:
           command:
             - /bin/agola
             - serve
-          - --embedded-etcd
             - "--config"
             - /mnt/agola/config/config.yml
             - "--components"