Added custom index for job status (#78)

* Added custom index for job status

.github/workflows/build.yml

@@ -0,0 +1,27 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - master
+
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    permissions: read-all
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - uses: sonarsource/sonarqube-scan-action@master
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+      # If you wish to fail your job when the Quality Gate is red, uncomment the
+      # following lines. This would typically be used to fail a deployment.
+      # - uses: sonarsource/sonarqube-quality-gate-action@master
+      #   timeout-minutes: 5
+      #   env:
+      #     SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/semicolons-kudos.yaml

@@ -0,0 +1,26 @@
+
+name: Kudos for Code
+on:
+  push:
+    branches: ["master"]
+  workflow_dispatch:
+
+jobs:
+  kudos:
+    name: Semicolons Kudos
+    permissions: write-all
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: LoremLabs/kudos-for-code-action@latest
+        with:
+          search-dir: "."
+          destination: "artifact"
+          generate-nomerges: true
+          generate-validemails: true
+          generate-limitdepth: 0
+          generate-fromrepo: true
+          analyze-repo: false
+          skip-ids: ""

CHANGELOG.md

@@ -0,0 +1,69 @@
+# Version 4.2.0
+* Added custom index for job status
+
+# Version 4.1.0
+* Added custom index for exception log
+
+# Version 4.0.0
+
+* Added Symfony 6 support
+* PHP minimum requirements bumped to 8.0
+
+# Version 3.1.0
+
+* Added optional "messenger mode", to delegate jobs execution to workers from the Symfony messenger component
+* Added support for asynchronous steps execution, using the AMPHP library (contribution from [matyo91](https://github.com/matyo91))
+
+# Version 3.0.0
+
+* Added PHP 8 support
+* PHP minimum requirements bumped to 7.3
+* Added Doctrine DBAL 3 support
+* Doctrine DBAL minimum requirements bumped to 2.12
+
+# Version 2.2.0
+
+* Improve logging Dataflow job
+
+# Version 2.1.1
+
+* Fixed some Symfony 5 compatibility issues
+
+# Version 2.1.0
+
+* Added CollectionWriter and DelegatorWriter
+* Adding Symfony 5.0 compatibility
+* Save all exceptions caught in the log for `code-rhapsodie:dataflow:execute`
+* Added more output when errors occured during `code-rhapsodie:dataflow:execute`
+
+# Version 2.0.2
+
+* Fixed the connection proxy class created by the factory
+
+# Version 2.0.1
+
+* Fixed next execution time not increasing for scheduled dataflows
+
+# Version 2.0.0
+
+* Add Doctrine DBAL multi-connection support
+* Add configuration to define the default Doctrine DBAL connection
+* Remove Doctrine ORM
+* Rewrite repositories
+
+# Version 1.0.1
+
+* Fix lost dependency
+* Fix schedule removing
+
+# Version 1.0.0
+
+Initial version
+
+* Define and configure a Dataflow
+* Run the Job scheduled
+* Run one Dataflow from the command line
+* Define the schedule for a Dataflow from the command line
+* Enable/Disable a scheduled Dataflow from the command line
+* Display the list of scheduled Dataflow from the command line
+* Display the result for the last Job for a Dataflow from the command line

README.md

@@ -1,8 +1,21 @@
-# Code-Rhapsodie Dataflow Bundle
+# Code Rhapsodie Dataflow Bundle
 
 DataflowBundle is a bundle for Symfony 3.4+ 
 providing an easy way to create import / export dataflow.
 
+Dataflow uses a linear generic workflow in three parts:
+ * one reader
+ * any number of steps that can be synchronous or asynchronous
+ * one or more writers
+
+The reader can read data from anywhere and return data row by row. Each step processes the current row data. 
+The steps are executed in the order in which they are added.
+And, one or more writers save the row anywhere you want.
+
+As the following schema shows, you can define more than one dataflow:
+
+![Dataflow schema](src/Resources/doc/schema.png)
+
 # Features
 
 * Define and configure a Dataflow
@@ -12,16 +25,19 @@ providing an easy way to create import / export dataflow.
 * Enable/Disable a scheduled Dataflow from the command line
 * Display the list of scheduled Dataflow from the command line
 * Display the result for the last Job for a Dataflow from the command line
+* Work with multiple Doctrine DBAL connections
 
 
 ## Installation
 
+Security notice: Symfony 4.x is not supported before 4.1.12, see https://github.com/advisories/GHSA-pgwj-prpq-jpc2
+
 ### Add the dependency
 
 To install this bundle, run this command :
 
 ```shell script
-$ composer require code-rhapsodie/dataflow
+$ composer require code-rhapsodie/dataflow-bundle
 ```
 
 #### Suggest
@@ -80,42 +96,81 @@ public function registerBundles()
 
 ### Update the database
 
-This bundle use Dotrine ORM for drive the database table for store Dataflow schedule (`cr_dataflow_scheduled`) 
+This bundle uses Doctrine DBAL to store Dataflow schedule into the database table (`cr_dataflow_scheduled`)
 and jobs (`cr_dataflow_job`).
 
-#### Doctrine migration
-
-Execute the command for generate the migration for your database:
-
-```shell script
-$ bin/console doctrine:migration:diff
-```
-
-#### Other migration tools
-
-If you use [Phinx](https://phinx.org/) or [Kaliop Migration Bundle](https://github.com/kaliop-uk/ezmigrationbundle) or whatever, 
+If you use [Doctrine Migration Bundle](https://symfony.com/doc/master/bundles/DoctrineMigrationsBundle/index.html) or [Phinx](https://phinx.org/) 
+or [Kaliop Migration Bundle](https://github.com/kaliop-uk/ezmigrationbundle) or whatever,
 you can add a new migration with the generated SQL query from this command:
 
 ```shell script
-$ bin/console doctrine:schema:update --dump-sql
+$ bin/console code-rhapsodie:dataflow:dump-schema
 ```
 
+If you have already the tables, you can add a new migration with the generated update SQL query from this command:
+
+```shell script
+$ bin/console code-rhapsodie:dataflow:dump-schema --update
+```
+
+## Configuration
+
+By default, the Doctrine DBAL connection used is `default`. You can configure the default connection.
+Add this configuration into your Symfony configuration:
+
+```yaml
+code_rhapsodie_dataflow:
+  dbal_default_connection: test #Name of the default connection used by Dataflow bundle
+```
+
+By default, the `logger` service will be used to log all exceptions and custom messages.
+If you want to use another logger, like a specific Monolog handler, Add this configuration:
+
+```yaml
+code_rhapsodie_dataflow:
+  default_logger: monolog.logger.custom #Service ID of the logger you want Dataflow to use
+```
+
+### Messenger mode
+
+Dataflow can delegate the execution of its jobs to the Symfony messenger component, if available.
+This allows jobs to be executed concurrently by workers instead of sequentially.
+
+To enable messenger mode:
+```yaml
+code_rhapsodie_dataflow:
+  messenger_mode:
+    enabled: true
+    # bus: 'messenger.default_bus' #Service ID of the bus you want Dataflow to use, if not the default one
+```
+
+You also need to route Dataflow messages to the proper transport:
+```yaml
+# config/packages/messenger.yaml
+framework:
+  messenger:
+    transports:
+      async: '%env(MESSENGER_TRANSPORT_DSN)%'
+
+    routing:
+      CodeRhapsodie\DataflowBundle\MessengerMode\JobMessage: async
+```
 
 ## Define a dataflow type
 
 This bundle uses a fixed and simple workflow structure in order to let you focus on the data processing logic part of your dataflow.
 
-A dataflow type defines the different parts of your dataflow. A dataflow is comprised of:
+A dataflow type defines the different parts of your dataflow. A dataflow is made of:
 - exactly one *Reader*
 - any number of *Steps*
 - one or more *Writers*
 
 Dataflow types can be configured with options.
 
-A dataflow type must implements `CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface`.
+A dataflow type must implement `CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface`.
 
-To help with creating your workflow types, an abstract class `CodeRhapsodie\DataflowBundle\DataflowType\AbstractDataflowType` 
-is provided, allowing you to define your dataflow through an handy builder `CodeRhapsodie\DataflowBundle\DataflowType\DataflowBuilder`.
+To help with creating your dataflow types, an abstract class `CodeRhapsodie\DataflowBundle\DataflowType\AbstractDataflowType`
+is provided, allowing you to define your dataflow through a handy builder `CodeRhapsodie\DataflowBundle\DataflowType\DataflowBuilder`.
 
 This is an example to define one class DataflowType:
 
@@ -142,11 +197,11 @@ class MyFirstDataflowType extends AbstractDataflowType
 
     protected function buildDataflow(DataflowBuilder $builder, array $options): void
     {
-        
-        $this->myReader->setFilename($options['fileName']);
+        $this->myWriter->setDestinationFilePath($options['to-file']);
 
-        $builder->setReader($this->myReader)
-            ->addStep(function($data) use ($options) {
+        $builder
+            ->setReader($this->myReader->read($options['from-file']))
+            ->addStep(function ($data) use ($options) {
                 // TODO : Write your code here...
                 return $data;
             })
@@ -156,11 +211,8 @@ class MyFirstDataflowType extends AbstractDataflowType
 
     protected function configureOptions(OptionsResolver $optionsResolver): void
     {
-        $optionsResolver->setDefaults([
-            'my_option' => 'my_default_value',
-            'fileName'  => null,
-        ]);
-        $optionsResolver->setRequired('fileName');
+        $optionsResolver->setDefaults(['to-file' => '/tmp/dataflow.csv', 'from-file' => null]);
+        $optionsResolver->setRequired('from-file');
     }
 
     public function getLabel(): string
@@ -176,18 +228,22 @@ class MyFirstDataflowType extends AbstractDataflowType
 
 ```
 
-The `DataflowTypeInterface` is used by Symfony for auto-configuration our custom datafow type only if the folder is correctly configured (see the `services` configuration file in your projet).
-If you don't use the auto-configuration, you must add this tag `coderhapsodie.dataflow.type` in your dataflow type service configuration:
+Dataflow types must be tagged with `coderhapsodie.dataflow.type`.
 
+If you're using Symfony auto-configuration for your services, this tag will be automatically added to all services implementing `DataflowTypeInterface`.
+
+Otherwise, manually add the tag `coderhapsodie.dataflow.type` in your dataflow type service configuration:
+
+```yaml
 ```yaml
     CodeRhapsodie\DataflowExemple\DataflowType\MyFirstDataflowType:
       tags:
         - { name: coderhapsodie.dataflow.type }
 ```
 
-### Use the options for your dataflow type
+### Use options for your dataflow type
 
-The `AbstractDataflowType` can help you define the options of your Datataflow type.
+The `AbstractDataflowType` can help you define options for your Dataflow type.
 
 Add this method in your DataflowType class:
 
@@ -201,11 +257,8 @@ class MyFirstDataflowType extends AbstractDataflowType
     // ...
     protected function configureOptions(OptionsResolver $optionsResolver): void
     {
-        $optionsResolver->setDefaults([
-            'my_option' => 'my_default_value',
-            'fileName'  => null,
-        ]);
-        $optionsResolver->setRequired('fileName');
+        $optionsResolver->setDefaults(['to-file' => '/tmp/dataflow.csv', 'from-file' => null]);
+        $optionsResolver->setRequired('from-file');
     }
 
 }
@@ -213,6 +266,33 @@ class MyFirstDataflowType extends AbstractDataflowType
 
 With this configuration, the option `fileName` is required. For an advanced usage of the option resolver, read the [Symfony documentation](https://symfony.com/doc/current/components/options_resolver.html).
 
+For asynchronous management, `AbstractDataflowType` come with two default options :
+- loopInterval : default to 0. Update this interval if you wish customise the `tick` loop duration.
+- emitInterval : default to 0. Update this interval to have a control when reader must emit new data in the flow pipeline.
+
+### Logging
+
+All exceptions will be caught and written in the logger.
+If you want to add custom messages in the log, you can inject the logger in your readers / steps / writers.
+If your DataflowType class extends `AbstractDataflowType`, the logger is accessible as `$this->logger`.
+
+```php
+<?php
+// ...
+use Symfony\Component\OptionsResolver\OptionsResolver;
+
+class MyDataflowType extends AbstractDataflowType
+{
+    // ...
+    protected function buildDataflow(DataflowBuilder $builder, array $options): void
+    {
+        $this->myWriter->setLogger($this->logger);
+    }
+
+}
+```
+
+When using the `code-rhapsodie:dataflow:run-pending` command, this logger will also be used to save the log in the corresponding job in the database.
 
 ### Check if your DataflowType is ready
 
@@ -239,9 +319,9 @@ Symfony Container Public and Private Services Tagged with "coderhapsodie.dataflo
 
 ### Readers
 
-*Readers* provide the workflow with elements to import / export. Usually, elements are read from an external resource (file, database, webservice, etc).
+*Readers* provide the dataflow with elements to import / export. Usually, elements are read from an external resource (file, database, webservice, etc).
 
-A *Reader* must implements `Port\Reader` or return a `iterable` if you use the `Port\Reader\IteratorReader`.
+A *Reader* can be any `iterable`.
 
 The only constraint on the returned elements typing is that they cannot be `false`.
 
@@ -254,36 +334,27 @@ namespace CodeRhapsodie\DataflowExemple\Reader;
 
 class FileReader
 {
-    private $filename;
-
-    /**
-     * Set the filename option needed by the Reader.
-     */
-    public function setFilename(string $filename) {
-        $this->filename = $filename;
-    }
-
-    public function __invoke(): iterable
+    public function read(string $filename): iterable
     {
-        if (!$this->filename) {
+        if (!$filename) {
             throw new \Exception("The file name is not defined. Define it with 'setFilename' method");
         }
 
-        if (!$fh = fopen($this->filename, 'r')) {
-            throw new \Exception("Unable to open file '".$this->filename."' for read.");
+        if (!$fh = fopen($filename, 'r')) {
+            throw new \Exception("Unable to open file '".$filename."' for read.");
         }
 
-        while (false === ($read = fread($fh, 1024))) {
-            yield explode("|", $read);
+        while (false !== ($read = fgets($fh))) {
+            yield explode('|', trim($read));
         }
     }
 }
 ```
 
-To setup your reader in the dataflow builder, you must use `Port\Reader\IteratorReader` like this
+You can set up this reader as follows:
 
 ```php
-$builder->setReader(new \Port\Reader\IteratorReader($this->myReader))
+$builder->setReader(($this->myReader)())
 ``` 
 
 
@@ -291,18 +362,53 @@ $builder->setReader(new \Port\Reader\IteratorReader($this->myReader))
 
 *Steps* are operations performed on the elements before they are handled by the *Writers*. Usually, steps are either:
 - converters, that alter the element
-- filters, that conditionally prevents further operations on the element
+- filters, that conditionally prevent further operations on the element
+- generators, that can include asynchronous operations
 
 A *Step* can be any callable, taking the element as its argument, and returning either:
 - the element, possibly altered
 - `false`, if no further operations should be performed on this element
 
+A few examples:
+
+```php
+<?php
+//[...]
+$builder->addStep(function ($item) {
+    // Titles are changed to all caps before export
+    $item['title'] = strtoupper($item['title']);
+
+    return $item;
+});
+
+// asynchronous step with 2 scale factor
+$builder->addStep(function ($item): \Generator {
+    yield new \Amp\Delayed(1000); // asynchronous processing for 1 second long
+
+    // Titles are changed to all caps before export
+    $item['title'] = strtolower($item['title']);
+
+    return $item;
+}, 2);
+
+$builder->addStep(function ($item) {
+    // Private items are not exported
+    if ($item['private']) {
+        return false;
+    }
+
+    return $item;
+});
+//[...]
+```
+
+Note : you can ensure writing order for asynchronous operations if all steps are scaled at 1 factor.
 
 ### Writers
 
-*Writers* performs the actual import / export operations.
+*Writers* perform the actual import / export operations.
 
-A *Writer* must implements `CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface`.
+A *Writer* must implement `CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface`.
 As this interface is not compatible with `Port\Writer`, the adapter `CodeRhapsodie\DataflowBundle\DataflowType\Writer\PortWriterAdapter` is provided.
 
 This example show how to use the predefined PhpPort Writer :
@@ -311,7 +417,7 @@ This example show how to use the predefined PhpPort Writer :
 $builder->addWriter(new PortWriterAdapter(new \Port\FileWriter()));
 ```
 
-Or you own Writer:
+Or your own Writer:
 
 ```php
 <?php
@@ -323,11 +429,20 @@ class FileWriter implements WriterInterface
 {
     private $fh;
 
+    /** @var string */
+    private $path;
+
+    public function setDestinationFilePath(string $path) {
+        $this->path = $path;
+    }
+
     public function prepare()
     {
-
-        if (!$this->fh = fopen('/path/to/file', 'w')) {
-            throw new \Exception("Unable to open in write mode the output file.");
+        if (null === $this->path) {
+            throw new \Exception('Define the destination file name before use');
+        }
+        if (!$this->fh = fopen($this->path, 'w')) {
+            throw new \Exception('Unable to open in write mode the output file.');
         }
     }
 
@@ -343,21 +458,112 @@ class FileWriter implements WriterInterface
 }
 ```
 
+#### CollectionWriter
+
+If you want to write multiple items from a single item read, you can use the generic `CollectionWriter`. This writer will iterate over any `iterable` it receives, and pass each item from that collection to your own writer that handles single items.
+
+```php
+$builder->addWriter(new CollectionWriter($mySingleItemWriter));
+```
+
+#### DelegatorWriter
+
+If you want to call different writers depending on what item is read, you can use the generic `DelegatorWriter`.
+
+As an example, let's suppose our items are arrays with the first entry being either `product` or `order`. We want to use a different writer based on that value.
+
+First, create your writers implementing `DelegateWriterInterface` (this interface extends `WriterInterface` so your writers can still be used without the `DelegatorWriter`).
+
+```php
+<?php
+namespace CodeRhapsodie\DataflowExemple\Writer;
+
+use CodeRhapsodie\DataFlowBundle\DataflowType\Writer\WriterInterface;
+
+class ProductWriter implements DelegateWriterInterface
+{
+    public function supports($item): bool
+    {
+        return 'product' === reset($item);
+    }
+
+    public function prepare()
+    {
+    }
+
+    public function write($item)
+    {
+        // Process your product
+    }
+
+    public function finish()
+    {
+    }
+}
+```
+
+```php
+<?php
+namespace CodeRhapsodie\DataflowExemple\Writer;
+
+use CodeRhapsodie\DataFlowBundle\DataflowType\Writer\WriterInterface;
+
+class OrderWriter implements DelegateWriterInterface
+{
+    public function supports($item): bool
+    {
+        return 'order' === reset($item);
+    }
+
+    public function prepare()
+    {
+    }
+
+    public function write($item)
+    {
+        // Process your order
+    }
+
+    public function finish()
+    {
+    }
+}
+```
+
+Then, configure your `DelegatorWriter` and add it to your dataflow type.
+
+```php
+    protected function buildDataflow(DataflowBuilder $builder, array $options): void
+    {
+        // Snip add reader and steps
+
+        $delegatorWriter = new DelegatorWriter();
+        $delegatorWriter->addDelegate(new ProductWriter());
+        $delegatorWriter->addDelegate(new OrderWriter());
+
+        $builder->addWriter($delegatorWriter);
+    }
+```
+
+During execution, the `DelegatorWriter` will simply pass each item received to its first delegate (in the order those were added) that supports it. If no delegate supports an item, an exception will be thrown.
+
 ## Queue
 
 All pending dataflow job processes are stored in a queue into the database.
 
-Add this command into your crontab for execute all queued job:
+Add this command into your crontab for execute all queued jobs:
 
 ```shell script
-$ SYMFONY_ENV=prod php bin/console code-rhapsodie:dataflow:job:run-pending
+$ SYMFONY_ENV=prod php bin/console code-rhapsodie:dataflow:run-pending
 ```
 
 ## Commands
 
-Many commands are provided.
+Several commands are provided to manage schedules and run jobs.
 
-`code-rhapsodie:dataflow:job:run-pending` Executes job in the queue according to their schedule.
+`code-rhapsodie:dataflow:run-pending` Executes job in the queue according to their schedule.
+
+When messenger mode is enabled, jobs will still be created according to their schedule, but execution will be handled by the messenger component instead.
 
 `code-rhapsodie:dataflow:schedule:list` Display the list of dataflows scheduled.
 
@@ -367,13 +573,37 @@ Many commands are provided.
 
 `code-rhapsodie:dataflow:job:show` Display the last result of a job.
 
-`code-rhapsodie:dataflow:execute` Lets you execute one dataflow job.
+`code-rhapsodie:dataflow:execute` Let you execute one dataflow job.
 
+`code-rhapsodie:dataflow:dump-schema` Generates schema create / update SQL queries
+
+### Work with many databases
+
+All commands have a `--connection` option to define what Doctrine DBAL connection to use during execution.
+
+Example:
+
+This command uses the `default` DBAL connection to generate all schema update queries.
+
+```shell script
+$ bin/console code-rhapsodie:dataflow:dump-schema --update --connection=default
+```
+
+To execute all pending job for a specific connection use:
+
+```shell script
+# Run for dataflow DBAL connection
+$ bin/console code-rhapsodie:dataflow:run-pending --connection=dataflow
+# Run for default DBAL connection
+$ bin/console code-rhapsodie:dataflow:run-pending --connection=default
+```
 
 # Issues and feature requests
 
 Please report issues and request features at https://github.com/code-rhapsodie/dataflow-bundle/issues.
 
+Please note that only the last release of the 3.x and the 4.x versions of this bundle are actively supported.
+
 # Contributing
 
 Contributions are very welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for

Tests/DataflowType/AbstractDataflowTypeTest.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType;
+
+use CodeRhapsodie\DataflowBundle\DataflowType\AbstractDataflowType;
+use CodeRhapsodie\DataflowBundle\DataflowType\DataflowBuilder;
+use PHPUnit\Framework\TestCase;
+use Symfony\Component\OptionsResolver\OptionsResolver;
+
+class AbstractDataflowTypeTest extends TestCase
+{
+    public function testProcess()
+    {
+        $label = 'Test label';
+        $options = ['testOption' => 'Test value'];
+        $values = [1, 2, 3];
+        $testCase = $this;
+
+        $dataflowType = new class($label, $options, $values, $testCase) extends AbstractDataflowType
+        {
+            public function __construct(private string $label, private array $options, private array $values, private TestCase $testCase)
+            {
+            }
+
+            public function getLabel(): string
+            {
+                return $this->label;
+            }
+
+            protected function configureOptions(OptionsResolver $optionsResolver): void
+            {
+                $optionsResolver->setDefined('testOption');
+            }
+
+            protected function buildDataflow(DataflowBuilder $builder, array $options): void
+            {
+                $builder->setReader($this->values);
+                $this->testCase->assertSame($this->options, $options);
+            }
+        };
+
+        $result = $dataflowType->process($options);
+        $this->assertSame($label, $result->getName());
+        $this->assertSame(count($values), $result->getTotalProcessedCount());
+    }
+}

Tests/DataflowType/Dataflow/AMPAsyncDataflowTest.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType\Dataflow;
+
+use Amp\Delayed;
+use CodeRhapsodie\DataflowBundle\DataflowType\Dataflow\AMPAsyncDataflow;
+use CodeRhapsodie\DataflowBundle\DataflowType\Dataflow\Dataflow;
+use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
+use PHPUnit\Framework\TestCase;
+
+class AMPAsyncDataflowTest extends TestCase
+{
+    public function testProcess()
+    {
+        $reader = [1, 2, 3];
+        $result = [];
+        $dataflow = new AMPAsyncDataflow($reader, 'simple');
+        $dataflow->addStep(static fn($item) => $item + 1);
+        $dataflow->addStep(static function($item): \Generator {
+            yield new Delayed(10); //delay 10 milliseconds
+            return $item * 2;
+        });
+        $dataflow->addWriter(new class($result) implements WriterInterface {
+            private $buffer;
+
+            public function __construct(&$buffer) {
+                $this->buffer = &$buffer;
+            }
+
+            public function prepare()
+            {
+            }
+
+            public function write($item)
+            {
+                $this->buffer[] = $item;
+            }
+
+            public function finish()
+            {
+            }
+        });
+        $dataflow->process();
+
+        self::assertSame([4, 6, 8], $result);
+    }
+}

Tests/DataflowType/Writer/CollectionWriterTest.php

@@ -0,0 +1,54 @@
+<?php
+
+namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType\Writer;
+
+use CodeRhapsodie\DataflowBundle\DataflowType\Writer\CollectionWriter;
+use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
+use CodeRhapsodie\DataflowBundle\Exceptions\UnsupportedItemTypeException;
+use PHPUnit\Framework\TestCase;
+
+class CollectionWriterTest extends TestCase
+{
+    public function testNotACollection()
+    {
+        $this->expectException(UnsupportedItemTypeException::class);
+
+        $writer = new CollectionWriter($this->createMock(WriterInterface::class));
+        $writer->write('Not an iterable');
+    }
+
+    public function testSupports()
+    {
+        $writer = new CollectionWriter($this->createMock(WriterInterface::class));
+
+        $this->assertTrue($writer->supports([]));
+        $this->assertTrue($writer->supports(new \ArrayIterator([])));
+        $this->assertFalse($writer->supports(''));
+        $this->assertFalse($writer->supports(0));
+    }
+
+    public function testAll()
+    {
+        $values = ['a', 'b', 'c'];
+
+        $embeddedWriter = $this->createMock(WriterInterface::class);
+        $embeddedWriter
+            ->expects($this->once())
+            ->method('prepare')
+        ;
+        $embeddedWriter
+            ->expects($this->once())
+            ->method('finish')
+        ;
+        $embeddedWriter
+            ->expects($this->exactly(count($values)))
+            ->method('write')
+            ->withConsecutive(...array_map(fn($item) => [$item], $values))
+        ;
+
+        $writer = new CollectionWriter($embeddedWriter);
+        $writer->prepare();
+        $writer->write($values);
+        $writer->finish();
+    }
+}

Tests/DataflowType/Writer/DelegatorWriterTest.php

@@ -0,0 +1,150 @@
+<?php
+
+namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType\Writer;
+
+use CodeRhapsodie\DataflowBundle\DataflowType\Writer\DelegateWriterInterface;
+use CodeRhapsodie\DataflowBundle\DataflowType\Writer\DelegatorWriter;
+use CodeRhapsodie\DataflowBundle\Exceptions\UnsupportedItemTypeException;
+use PHPUnit\Framework\MockObject\MockObject;
+use PHPUnit\Framework\TestCase;
+
+class DelegatorWriterTest extends TestCase
+{
+    private \CodeRhapsodie\DataflowBundle\DataflowType\Writer\DelegatorWriter $delegatorWriter;
+
+    private \CodeRhapsodie\DataflowBundle\DataflowType\Writer\DelegateWriterInterface|\PHPUnit\Framework\MockObject\MockObject $delegateInt;
+
+    private \CodeRhapsodie\DataflowBundle\DataflowType\Writer\DelegateWriterInterface|\PHPUnit\Framework\MockObject\MockObject $delegateString;
+
+    private \CodeRhapsodie\DataflowBundle\DataflowType\Writer\DelegateWriterInterface|\PHPUnit\Framework\MockObject\MockObject $delegateArray;
+
+    protected function setUp(): void
+    {
+        $this->delegateInt = $this->createMock(DelegateWriterInterface::class);
+        $this->delegateInt->method('supports')->willReturnCallback(fn($argument) => is_int($argument));
+
+        $this->delegateString = $this->createMock(DelegateWriterInterface::class);
+        $this->delegateString->method('supports')->willReturnCallback(fn($argument) => is_string($argument));
+
+        $this->delegateArray = $this->createMock(DelegateWriterInterface::class);
+        $this->delegateArray->method('supports')->willReturnCallback(fn($argument) => is_array($argument));
+
+        $this->delegatorWriter = new DelegatorWriter();
+        $this->delegatorWriter->addDelegates([
+            $this->delegateInt,
+            $this->delegateString,
+            $this->delegateArray,
+        ]);
+    }
+
+    public function testUnsupported()
+    {
+        $this->expectException(UnsupportedItemTypeException::class);
+
+        $this->delegatorWriter->write(new \stdClass());
+    }
+
+    public function testStopAtFirstSupportingDelegate()
+    {
+        $value = 0;
+
+        $this->delegateInt->expects($this->once())->method('supports');
+        $this->delegateInt
+            ->expects($this->once())
+            ->method('write')
+            ->with($value)
+        ;
+        $this->delegateString->expects($this->never())->method('supports');
+        $this->delegateArray->expects($this->never())->method('supports');
+        $this->delegateString->expects($this->never())->method('write');
+        $this->delegateArray->expects($this->never())->method('write');
+
+        $this->delegatorWriter->write($value);
+    }
+
+    public function testNotSupported()
+    {
+        $value = new \stdClass();
+
+        $this->delegateInt
+            ->expects($this->once())
+            ->method('supports')
+            ->with($value)
+        ;
+        $this->delegateString
+            ->expects($this->once())
+            ->method('supports')
+            ->with($value)
+        ;
+        $this->delegateArray
+            ->expects($this->once())
+            ->method('supports')
+            ->with($value)
+        ;
+
+        $this->assertFalse($this->delegatorWriter->supports($value));
+    }
+
+    public function testSupported()
+    {
+        $value = '';
+
+        $this->delegateInt
+            ->expects($this->once())
+            ->method('supports')
+            ->with($value)
+        ;
+        $this->delegateString
+            ->expects($this->once())
+            ->method('supports')
+            ->with($value)
+        ;
+        $this->delegateArray
+            ->expects($this->never())
+            ->method('supports')
+        ;
+
+        $this->assertTrue($this->delegatorWriter->supports($value));
+    }
+
+    public function testAll()
+    {
+        $value = ['a'];
+
+        $this->delegateInt
+            ->expects($this->once())
+            ->method('supports')
+            ->with($value)
+        ;
+        $this->delegateString
+            ->expects($this->once())
+            ->method('supports')
+            ->with($value)
+        ;
+        $this->delegateArray
+            ->expects($this->once())
+            ->method('supports')
+            ->with($value)
+        ;
+
+        $this->delegateInt->expects($this->once())->method('prepare');
+        $this->delegateString->expects($this->once())->method('prepare');
+        $this->delegateArray->expects($this->once())->method('prepare');
+
+        $this->delegateInt->expects($this->once())->method('finish');
+        $this->delegateString->expects($this->once())->method('finish');
+        $this->delegateArray->expects($this->once())->method('finish');
+
+        $this->delegateInt->expects($this->never())->method('write');
+        $this->delegateString->expects($this->never())->method('write');
+        $this->delegateArray
+            ->expects($this->once())
+            ->method('write')
+            ->with($value)
+        ;
+
+        $this->delegatorWriter->prepare();
+        $this->delegatorWriter->write($value);
+        $this->delegatorWriter->finish();
+    }
+}

Tests/DataflowType/Writer/PortWriterAdapterTest.php

@@ -0,0 +1,37 @@
+<?php
+
+namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType\Writer;
+
+use CodeRhapsodie\DataflowBundle\DataflowType\Writer\PortWriterAdapter;
+use PHPUnit\Framework\TestCase;
+
+class PortWriterAdapterTest extends TestCase
+{
+    public function testAll()
+    {
+        $value = 'not an array';
+
+        $writer = $this->getMockBuilder('\Port\Writer')
+            ->setMethods(['prepare', 'finish', 'writeItem'])
+            ->getMock()
+        ;
+        $writer
+            ->expects($this->once())
+            ->method('prepare')
+        ;
+        $writer
+            ->expects($this->once())
+            ->method('finish')
+        ;
+        $writer
+            ->expects($this->once())
+            ->method('writeItem')
+            ->with([$value])
+        ;
+
+        $adapter = new PortWriterAdapter($writer);
+        $adapter->prepare();
+        $adapter->write($value);
+        $adapter->finish();
+    }
+}

Tests/Manager/ScheduledDataflowManagerTest.php

@@ -9,37 +9,34 @@ use CodeRhapsodie\DataflowBundle\Exceptions\UnknownDataflowTypeException;
 use CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManager;
 use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
 use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
-use Doctrine\ORM\EntityManagerInterface;
+use Doctrine\DBAL\Connection;
 use PHPUnit\Framework\MockObject\MockObject;
 use PHPUnit\Framework\TestCase;
 
 class ScheduledDataflowManagerTest extends TestCase
 {
-    /** @var ScheduledDataflowManager */
-    private $manager;
+    private \CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManager $manager;
 
-    /** @var EntityManagerInterface|MockObject */
-    private $em;
+    private \Doctrine\DBAL\Connection|\PHPUnit\Framework\MockObject\MockObject $connection;
 
-    /** @var ScheduledDataflowRepository|MockObject */
-    private $scheduledDataflowRepository;
+    private \CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository|\PHPUnit\Framework\MockObject\MockObject $scheduledDataflowRepository;
 
-    /** @var JobRepository|MockObject */
-    private $jobRepository;
+    private \CodeRhapsodie\DataflowBundle\Repository\JobRepository|\PHPUnit\Framework\MockObject\MockObject $jobRepository;
 
     protected function setUp(): void
     {
-        $this->em = $this->createMock(EntityManagerInterface::class);
+        $this->connection = $this->createMock(Connection::class);
         $this->scheduledDataflowRepository = $this->createMock(ScheduledDataflowRepository::class);
         $this->jobRepository = $this->createMock(JobRepository::class);
 
-        $this->manager = new ScheduledDataflowManager($this->em, $this->scheduledDataflowRepository, $this->jobRepository);
+        $this->manager = new ScheduledDataflowManager($this->connection, $this->scheduledDataflowRepository, $this->jobRepository);
     }
 
     public function testCreateJobsFromScheduledDataflows()
     {
         $scheduled1 = new ScheduledDataflow();
         $scheduled2 = (new ScheduledDataflow())
+            ->setId(-1)
             ->setDataflowType($type = 'testType')
             ->setOptions($options = ['opt' => 'val'])
             ->setNext($next = new \DateTime())
@@ -60,30 +57,76 @@ class ScheduledDataflowManagerTest extends TestCase
             ->willReturnOnConsecutiveCalls(new Job(), null)
         ;
 
-        $this->em
+        $this->connection
             ->expects($this->once())
-            ->method('persist')
+            ->method('beginTransaction')
+        ;
+        $this->jobRepository
+            ->expects($this->once())
+            ->method('save')
             ->with(
-                $this->callback(function (Job $job) use ($type, $options, $next, $label, $scheduled2) {
-                    return (
-                        $job->getStatus() === Job::STATUS_PENDING
-                        && $job->getDataflowType() === $type
-                        && $job->getOptions() === $options
-                        && $job->getRequestedDate() == $next
-                        && $job->getLabel() === $label
-                        && $job->getScheduledDataflow() === $scheduled2
-                    );
-                })
+                $this->callback(fn(Job $job) => $job->getStatus() === Job::STATUS_PENDING
+                && $job->getDataflowType() === $type
+                && $job->getOptions() === $options
+                && $job->getRequestedDate() == $next
+                && $job->getLabel() === $label
+                && $job->getScheduledDataflowId() === $scheduled2->getId())
             )
         ;
 
-        $this->em
+        $this->scheduledDataflowRepository
             ->expects($this->once())
-            ->method('flush')
+            ->method('save')
+            ->with($scheduled2)
+        ;
+
+        $this->connection
+            ->expects($this->once())
+            ->method('commit')
         ;
 
         $this->manager->createJobsFromScheduledDataflows();
 
         $this->assertEquals($next->add(\DateInterval::createFromDateString($frequency)), $scheduled2->getNext());
     }
+
+    public function testCreateJobsFromScheduledDataflowsWithError()
+    {
+        $scheduled1 = new ScheduledDataflow();
+
+        $this->scheduledDataflowRepository
+            ->expects($this->once())
+            ->method('findReadyToRun')
+            ->willReturn([$scheduled1])
+        ;
+
+        $this->jobRepository
+            ->expects($this->exactly(1))
+            ->method('findPendingForScheduledDataflow')
+            ->withConsecutive([$scheduled1])
+            ->willThrowException(new \Exception())
+        ;
+
+        $this->connection
+            ->expects($this->once())
+            ->method('beginTransaction')
+        ;
+        $this->jobRepository
+            ->expects($this->never())
+            ->method('save')
+        ;
+
+        $this->connection
+            ->expects($this->never())
+            ->method('commit')
+        ;
+        $this->connection
+            ->expects($this->once())
+            ->method('rollBack')
+        ;
+
+        $this->expectException(\Exception::class);
+
+        $this->manager->createJobsFromScheduledDataflows();
+    }
 }

Tests/MessengerMode/JobMessageHandlerTest.php

@@ -0,0 +1,55 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Tests\MessengerMode;
+
+use CodeRhapsodie\DataflowBundle\Entity\Job;
+use CodeRhapsodie\DataflowBundle\MessengerMode\JobMessage;
+use CodeRhapsodie\DataflowBundle\MessengerMode\JobMessageHandler;
+use CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface;
+use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
+use PHPUnit\Framework\MockObject\MockObject;
+use PHPUnit\Framework\TestCase;
+
+class JobMessageHandlerTest extends TestCase
+{
+    private \CodeRhapsodie\DataflowBundle\Repository\JobRepository|\PHPUnit\Framework\MockObject\MockObject $repository;
+
+    private \CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface|\PHPUnit\Framework\MockObject\MockObject $processor;
+
+    private \CodeRhapsodie\DataflowBundle\MessengerMode\JobMessageHandler $handler;
+
+    protected function setUp(): void
+    {
+        $this->repository = $this->createMock(JobRepository::class);
+        $this->processor = $this->createMock(JobProcessorInterface::class);
+
+        $this->handler = new JobMessageHandler($this->repository, $this->processor);
+    }
+
+    public function testGetHandledMessages()
+    {
+        $this->assertSame([JobMessage::class], JobMessageHandler::getHandledMessages());
+    }
+
+    public function testInvoke()
+    {
+        $message = new JobMessage($id = 32);
+
+        $this->repository
+            ->expects($this->once())
+            ->method('find')
+            ->with($id)
+            ->willReturn($job = new Job())
+        ;
+
+        $this->processor
+            ->expects($this->once())
+            ->method('process')
+            ->with($job)
+        ;
+
+        ($this->handler)($message);
+    }
+}

Tests/Processor/JobProcessorTest.php

@@ -0,0 +1,108 @@
+<?php
+
+namespace CodeRhapsodie\DataflowBundle\Tests\Processor;
+
+use CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface;
+use CodeRhapsodie\DataflowBundle\DataflowType\Result;
+use CodeRhapsodie\DataflowBundle\Entity\Job;
+use CodeRhapsodie\DataflowBundle\Event\Events;
+use CodeRhapsodie\DataflowBundle\Event\ProcessingEvent;
+use CodeRhapsodie\DataflowBundle\Processor\JobProcessor;
+use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface;
+use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
+use PHPUnit\Framework\MockObject\MockObject;
+use PHPUnit\Framework\TestCase;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+
+class JobProcessorTest extends TestCase
+{
+    private \CodeRhapsodie\DataflowBundle\Processor\JobProcessor $processor;
+
+    private \CodeRhapsodie\DataflowBundle\Repository\JobRepository|\PHPUnit\Framework\MockObject\MockObject $repository;
+
+    private \CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface|\PHPUnit\Framework\MockObject\MockObject $registry;
+
+    private \Symfony\Component\EventDispatcher\EventDispatcherInterface|\PHPUnit\Framework\MockObject\MockObject $dispatcher;
+
+    protected function setUp(): void
+    {
+        $this->repository = $this->createMock(JobRepository::class);
+        $this->registry = $this->createMock(DataflowTypeRegistryInterface::class);
+        $this->dispatcher = $this->createMock(EventDispatcherInterface::class);
+
+        $this->processor = new JobProcessor($this->repository, $this->registry, $this->dispatcher);
+    }
+
+    public function testProcess()
+    {
+        $now = new \DateTimeImmutable();
+        $job = (new Job())
+            ->setStatus(Job::STATUS_PENDING)
+            ->setDataflowType($type = 'type')
+            ->setOptions($options = ['option1' => 'value1'])
+        ;
+
+        // Symfony 3.4 to 4.4 call
+        if (!class_exists(\Symfony\Contracts\EventDispatcher\Event::class)) {
+            $this->dispatcher
+                ->expects($this->exactly(2))
+                ->method('dispatch')
+                ->withConsecutive(
+                    [
+                        Events::BEFORE_PROCESSING,
+                        $this->callback(fn(ProcessingEvent $event) => $event->getJob() === $job)
+                    ],
+                    [
+                        Events::AFTER_PROCESSING,
+                        $this->callback(fn(ProcessingEvent $event) => $event->getJob() === $job)
+                    ],
+                );
+        } else { // Symfony 5.0+
+            $this->dispatcher
+                ->expects($this->exactly(2))
+                ->method('dispatch')
+                ->withConsecutive(
+                    [
+                        $this->callback(fn(ProcessingEvent $event) => $event->getJob() === $job),
+                        Events::BEFORE_PROCESSING,
+                    ],
+                    [
+                        $this->callback(fn(ProcessingEvent $event) => $event->getJob() === $job),
+                        Events::AFTER_PROCESSING,
+                    ],
+                );
+        }
+
+        $dataflowType = $this->createMock(DataflowTypeInterface::class);
+
+        $this->registry
+            ->expects($this->once())
+            ->method('getDataflowType')
+            ->with($type)
+            ->willReturn($dataflowType)
+        ;
+
+        $bag = [new \Exception('message1')];
+
+        $result = new Result('name', new \DateTimeImmutable(), $end = new \DateTimeImmutable(), $count = 10, $bag);
+
+        $dataflowType
+            ->expects($this->once())
+            ->method('process')
+            ->with($options)
+            ->willReturn($result)
+        ;
+
+        $this->repository
+            ->expects($this->exactly(2))
+            ->method('save')
+        ;
+
+        $this->processor->process($job);
+
+        $this->assertGreaterThanOrEqual($now, $job->getStartTime());
+        $this->assertSame(Job::STATUS_COMPLETED, $job->getStatus());
+        $this->assertSame($end, $job->getEndTime());
+        $this->assertSame($count - count($bag), $job->getCount());
+    }
+}

Tests/Registry/DataflowTypeRegistryTest.php

@@ -10,8 +10,7 @@ use PHPUnit\Framework\TestCase;
 
 class DataflowTypeRegistryTest extends TestCase
 {
-    /** @var DataflowTypeRegistry */
-    private $registry;
+    private \CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistry $registry;
 
     protected function setUp(): void
     {
@@ -33,7 +32,7 @@ class DataflowTypeRegistryTest extends TestCase
 
         $this->registry->registerDataflowType($type);
 
-        $this->assertSame($type, $this->registry->getDataflowType(get_class($type)));
+        $this->assertSame($type, $this->registry->getDataflowType($type::class));
         $this->assertSame($type, $this->registry->getDataflowType($alias1));
         $this->assertSame($type, $this->registry->getDataflowType($alias2));
         $this->assertContains($type, $this->registry->listDataflowTypes());

Tests/Runner/MessengerDataflowRunnerTest.php

@@ -0,0 +1,65 @@
+<?php
+
+namespace CodeRhapsodie\DataflowBundle\Tests\Runner;
+
+use CodeRhapsodie\DataflowBundle\Entity\Job;
+use CodeRhapsodie\DataflowBundle\MessengerMode\JobMessage;
+use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
+use CodeRhapsodie\DataflowBundle\Runner\MessengerDataflowRunner;
+use PHPUnit\Framework\MockObject\MockObject;
+use PHPUnit\Framework\TestCase;
+use Symfony\Component\Messenger\Envelope;
+use Symfony\Component\Messenger\MessageBusInterface;
+
+class MessengerDataflowRunnerTest extends TestCase
+{
+    private \CodeRhapsodie\DataflowBundle\Runner\MessengerDataflowRunner $runner;
+
+    private \CodeRhapsodie\DataflowBundle\Repository\JobRepository|\PHPUnit\Framework\MockObject\MockObject $repository;
+
+    private \Symfony\Component\Messenger\MessageBusInterface|\PHPUnit\Framework\MockObject\MockObject $bus;
+
+    protected function setUp(): void
+    {
+        $this->repository = $this->createMock(JobRepository::class);
+        $this->bus = $this->createMock(MessageBusInterface::class);
+
+        $this->runner = new MessengerDataflowRunner($this->repository, $this->bus);
+    }
+
+    public function testRunPendingDataflows()
+    {
+        $job1 = (new Job())->setId($id1 = 10);
+        $job2 = (new Job())->setId($id2 = 20);
+
+        $this->repository
+            ->expects($this->exactly(3))
+            ->method('findNextPendingDataflow')
+            ->willReturnOnConsecutiveCalls($job1, $job2, null)
+        ;
+        $this->repository
+            ->expects($this->exactly(2))
+            ->method('save')
+            ->withConsecutive([$job1], [$job2])
+        ;
+
+        $this->bus
+            ->expects($this->exactly(2))
+            ->method('dispatch')
+            ->withConsecutive([
+                $this->callback(fn($message) => $message instanceof JobMessage && $message->getJobId() === $id1)
+            ], [
+                $this->callback(fn($message) => $message instanceof JobMessage && $message->getJobId() === $id2)
+            ])
+            ->willReturnOnConsecutiveCalls(
+                new Envelope(new JobMessage($id1)),
+                new Envelope(new JobMessage($id2))
+            )
+        ;
+
+        $this->runner->runPendingDataflows();
+
+        $this->assertSame(Job::STATUS_QUEUED, $job1->getStatus());
+        $this->assertSame(Job::STATUS_QUEUED, $job2->getStatus());
+    }
+}

Tests/Runner/PendingDataflowRunnerTest.php

@@ -2,59 +2,33 @@
 
 namespace CodeRhapsodie\DataflowBundle\Tests\Runner;
 
-use CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface;
-use CodeRhapsodie\DataflowBundle\DataflowType\Result;
 use CodeRhapsodie\DataflowBundle\Entity\Job;
-use CodeRhapsodie\DataflowBundle\Event\Events;
-use CodeRhapsodie\DataflowBundle\Event\ProcessingEvent;
-use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface;
+use CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface;
 use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
 use CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunner;
-use Doctrine\ORM\EntityManagerInterface;
 use PHPUnit\Framework\MockObject\MockObject;
 use PHPUnit\Framework\TestCase;
-use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 
 class PendingDataflowRunnerTest extends TestCase
 {
-    /** @var PendingDataflowRunner */
-    private $runner;
+    private \CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunner $runner;
 
-    /** @var EntityManagerInterface|MockObject */
-    private $em;
+    private \CodeRhapsodie\DataflowBundle\Repository\JobRepository|\PHPUnit\Framework\MockObject\MockObject $repository;
 
-    /** @var JobRepository|MockObject */
-    private $repository;
-
-    /** @var DataflowTypeRegistryInterface|MockObject */
-    private $registry;
-
-    /** @var EventDispatcherInterface|MockObject */
-    private $dispatcher;
+    private \CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface|\PHPUnit\Framework\MockObject\MockObject $processor;
 
     protected function setUp(): void
     {
-        $this->em = $this->createMock(EntityManagerInterface::class);
         $this->repository = $this->createMock(JobRepository::class);
-        $this->registry = $this->createMock(DataflowTypeRegistryInterface::class);
-        $this->dispatcher = $this->createMock(EventDispatcherInterface::class);
+        $this->processor = $this->createMock(JobProcessorInterface::class);
 
-        $this->runner = new PendingDataflowRunner($this->em, $this->repository, $this->registry, $this->dispatcher);
+        $this->runner = new PendingDataflowRunner($this->repository, $this->processor);
     }
 
     public function testRunPendingDataflows()
     {
-        $now = new \DateTime();
-        $job1 = (new Job())
-            ->setStatus(Job::STATUS_PENDING)
-            ->setDataflowType($type1 = 'type1')
-            ->setOptions($options1 = ['option1' => 'value1'])
-        ;
-        $job2 = (new Job())
-            ->setStatus(Job::STATUS_PENDING)
-            ->setDataflowType($type2 = 'type2')
-            ->setOptions($options2 = ['option2' => 'value2'])
-        ;
+        $job1 = new Job();
+        $job2 = new Job();
 
         $this->repository
             ->expects($this->exactly(3))
@@ -62,83 +36,12 @@ class PendingDataflowRunnerTest extends TestCase
             ->willReturnOnConsecutiveCalls($job1, $job2, null)
         ;
 
-        $this->dispatcher
-            ->expects($this->exactly(4))
-            ->method('dispatch')
-            ->withConsecutive(
-                [
-                    Events::BEFORE_PROCESSING,
-                    $this->callback(function (ProcessingEvent $event) use ($job1) {
-                        return $event->getJob() === $job1;
-                    })
-                ],
-                [
-                    Events::AFTER_PROCESSING,
-                    $this->callback(function (ProcessingEvent $event) use ($job1) {
-                        return $event->getJob() === $job1;
-                    })
-                ],
-                [
-                    Events::BEFORE_PROCESSING,
-                    $this->callback(function (ProcessingEvent $event) use ($job2) {
-                        return $event->getJob() === $job2;
-                    })
-                ],
-                [
-                    Events::AFTER_PROCESSING,
-                    $this->callback(function (ProcessingEvent $event) use ($job2) {
-                        return $event->getJob() === $job2;
-                    })
-                ]
-            )
-        ;
-
-        $dataflowType1 = $this->createMock(DataflowTypeInterface::class);
-        $dataflowType2 = $this->createMock(DataflowTypeInterface::class);
-
-        $this->registry
+        $this->processor
             ->expects($this->exactly(2))
-            ->method('getDataflowType')
-            ->withConsecutive([$type1], [$type2])
-            ->willReturnOnConsecutiveCalls($dataflowType1, $dataflowType2)
-        ;
-
-        $bag1 = new \SplObjectStorage();
-        $bag1->attach(new \Exception('message1'));
-        $bag2 = new \SplObjectStorage();
-        $bag2->attach(new \Exception('message2'));
-
-        $result1 = new Result('name', new \DateTime(), $end1 = new \DateTime(), $count1 = 10, $bag1);
-        $result2 = new Result('name', new \DateTime(), $end2 = new \DateTime(), $count2 = 20, $bag2);
-
-        $dataflowType1
-            ->expects($this->once())
             ->method('process')
-            ->with($options1)
-            ->willReturn($result1)
-        ;
-        $dataflowType2
-            ->expects($this->once())
-            ->method('process')
-            ->with($options2)
-            ->willReturn($result2)
-        ;
-
-        $this->em
-            ->expects($this->exactly(4))
-            ->method('flush')
+            ->withConsecutive([$job1], [$job2])
         ;
 
         $this->runner->runPendingDataflows();
-
-        $this->assertGreaterThanOrEqual($now, $job1->getStartTime());
-        $this->assertSame(Job::STATUS_COMPLETED, $job1->getStatus());
-        $this->assertSame($end1, $job1->getEndTime());
-        $this->assertSame($count1 - count($bag1), $job1->getCount());
-
-        $this->assertGreaterThanOrEqual($now, $job2->getStartTime());
-        $this->assertSame(Job::STATUS_COMPLETED, $job2->getStatus());
-        $this->assertSame($end2, $job2->getEndTime());
-        $this->assertSame($count2 - count($bag2), $job2->getCount());
     }
 }

Tests/Validator/Constraints/FrequencyValidatorTest.php

@@ -36,7 +36,7 @@ class FrequencyValidatorTest extends ConstraintValidatorTestCase
     public function testInvalidValue()
     {
         $constraint = new Frequency([
-            'invalidMessage' => 'testMessage',
+            'message' => 'testMessage',
         ]);
 
         $this->validator->validate('wrong value', $constraint);
@@ -53,7 +53,7 @@ class FrequencyValidatorTest extends ConstraintValidatorTestCase
     public function testNegativeIntervals($value)
     {
         $constraint = new Frequency([
-            'negativeIntervalMessage' => 'testMessage',
+            'message' => 'testMessage',
         ]);
 
         $this->validator->validate($value, $constraint);

UPGRADE.md

@@ -0,0 +1,3 @@
+# Upgrade from v1.x to v2.0
+
+[BC] `JobRepository` and `ScheduledDataflowRepository` are no longer a Doctrine ORM repository.

composer.json

@@ -2,7 +2,12 @@
     "name": "code-rhapsodie/dataflow-bundle",
     "description": "Data processing framework inspired by PortPHP",
     "type": "symfony-bundle",
-    "keywords": ["dataflow", "import", "export", "data processing"],
+    "keywords": [
+        "dataflow",
+        "import",
+        "export",
+        "data processing"
+    ],
     "license": "MIT",
     "authors": [
         {
@@ -36,22 +41,42 @@
         }
     },
     "require": {
-        "php": "^7.1",
-        "symfony/dependency-injection": "^3.4||^4.0",
-        "symfony/http-kernel": "^3.4||^4.0",
-        "symfony/config": "^3.4||^4.0",
-        "symfony/yaml": "^3.4||^4.0",
-        "symfony/console": "^3.4||^4.0",
-        "symfony/lock": "^3.4||^4.0",
-        "symfony/event-dispatcher": "^3.4||^4.0",
-        "symfony/validator": "^3.4||^4.0",
-        "symfony/options-resolver": "^3.4||^4.0",
-        "doctrine/orm": "^2.4.5"
+        "php": "^8.0",
+        "ext-json": "*",
+        "doctrine/dbal": "^2.12||^3.0",
+        "doctrine/doctrine-bundle": "^1.0||^2.0",
+        "monolog/monolog": "^1.0||^2.0",
+        "psr/log": "^1.1||^2.0||^3.0",
+        "symfony/config": "^3.4||^4.0||^5.0||^6.0",
+        "symfony/console": "^3.4||^4.0||^5.0||^6.0",
+        "symfony/dependency-injection": "^3.4||>=4.1.12||^5.0||^6.0",
+        "symfony/event-dispatcher": "^3.4||^4.0||^5.0||^6.0",
+        "symfony/http-kernel": "^3.4||^4.0||^5.0||^6.0",
+        "symfony/lock": "^3.4||^4.0||^5.0||^6.0",
+        "symfony/monolog-bridge": "^3.4||^4.0||^5.0||^6.0",
+        "symfony/options-resolver": "^3.4||^4.0||^5.0||^6.0",
+        "symfony/validator": "^3.4||^4.0||^5.0||^6.0",
+        "symfony/yaml": "^3.4||^4.0||^5.0||^6.0"
     },
     "require-dev": {
-        "phpunit/phpunit": "^8.1"
+        "amphp/amp": "^2.5",
+        "phpunit/phpunit": "^7||^8||^9",
+        "rector/rector": "^0.13.10",
+        "symfony/messenger": "^4.4||^5.0||^6.0"
     },
     "suggest": {
-        "portphp/portphp": "Provides generic readers, steps and writers for your dataflows."
+        "amphp/amp": "Provide asynchronous steps for your dataflows",
+        "portphp/portphp": "Provides generic readers, steps and writers for your dataflows.",
+        "symfony/messenger": "Allows messenger mode, i.e. letting workers run jobs"
+    },
+    "config": {
+        "sort-packages": true
+    },
+    "extra": {
+        "branch-alias": {
+            "dev-master": "2.1.x-dev",
+            "dev-v2.0.x": "2.0.x-dev",
+            "dev-v1.x": "1.x-dev"
+        }
     }
 }

phpunit.xml

@@ -1,32 +1,21 @@
-<?xml version = '1.0' encoding = 'UTF-8'?>
-
+<?xml version="1.0" encoding="UTF-8"?>
 <!-- http://www.phpunit.de/manual/current/en/appendixes.configuration.html -->
-<phpunit
-        backupGlobals="false"
-        backupStaticAttributes="false"
-        bootstrap="Tests/bootstrap.php"
-        convertErrorsToExceptions="true"
-        convertNoticesToExceptions="true"
-        convertWarningsToExceptions="true"
-        colors="false"
->
-    <php>
-        <ini name="error_reporting" value="-1" />
-    </php>
-    <testsuites>
-        <testsuite name="Port tests suite">
-            <directory suffix="Test.php">./Tests</directory>
-        </testsuite>
-    </testsuites>
-
-    <filter>
-        <whitelist>
-            <directory>./src/</directory>
-            <exclude>
-                <directory>Tests/</directory>
-                <directory>vendor/</directory>
-            </exclude>
-        </whitelist>
-    </filter>
-
+<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" backupGlobals="false" backupStaticAttributes="false" bootstrap="Tests/bootstrap.php" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" colors="false" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/9.3/phpunit.xsd">
+  <coverage>
+    <include>
+      <directory>./src/</directory>
+    </include>
+    <exclude>
+      <directory>Tests/</directory>
+      <directory>vendor/</directory>
+    </exclude>
+  </coverage>
+  <php>
+    <ini name="error_reporting" value="-1"/>
+  </php>
+  <testsuites>
+    <testsuite name="Dataflow tests suite">
+      <directory suffix="Test.php">./Tests</directory>
+    </testsuite>
+  </testsuites>
 </phpunit>

rector.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+use Rector\CodeQuality\Rector\Class_\InlineConstructorDefaultToPropertyRector;
+use Rector\Config\RectorConfig;
+use Rector\Set\ValueObject\LevelSetList;
+use Rector\Symfony\Set\SymfonySetList;
+
+return static function (RectorConfig $rectorConfig): void {
+    $rectorConfig->paths([
+        __DIR__ . '/src',
+        __DIR__ . '/Tests',
+    ]);
+
+    // register a single rule
+    $rectorConfig->rule(InlineConstructorDefaultToPropertyRector::class);
+
+    $rectorConfig->sets([
+        SymfonySetList::SYMFONY_60,
+        SymfonySetList::SYMFONY_CODE_QUALITY,
+        SymfonySetList::SYMFONY_CONSTRUCTOR_INJECTION,
+        LevelSetList::UP_TO_PHP_80,
+    ]);
+};

sonar-project.properties

@@ -0,0 +1,4 @@
+sonar.projectKey=code-rhapsodie_dataflow-bundle_AYvYuaAwWE9sbcQmD1vw
+
+sonar.sources=src
+sonar.tests=Tests

src/CodeRhapsodieDataflowBundle.php

@@ -5,21 +5,31 @@ declare(strict_types=1);
 namespace CodeRhapsodie\DataflowBundle;
 
 use CodeRhapsodie\DataflowBundle\DependencyInjection\CodeRhapsodieDataflowExtension;
+use CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler\BusCompilerPass;
 use CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler\DataflowTypeCompilerPass;
+use CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler\DefaultLoggerCompilerPass;
 use Symfony\Component\DependencyInjection\ContainerBuilder;
+use Symfony\Component\DependencyInjection\Extension\ExtensionInterface;
 use Symfony\Component\HttpKernel\Bundle\Bundle;
 
+/**
+ * @codeCoverageIgnore
+ */
 class CodeRhapsodieDataflowBundle extends Bundle
 {
     protected $name = 'CodeRhapsodieDataflowBundle';
 
-    public function getContainerExtension()
+    public function getContainerExtension(): ?ExtensionInterface
     {
         return new CodeRhapsodieDataflowExtension();
     }
 
     public function build(ContainerBuilder $container)
     {
-        $container->addCompilerPass(new DataflowTypeCompilerPass());
+        $container
+            ->addCompilerPass(new DataflowTypeCompilerPass())
+            ->addCompilerPass(new DefaultLoggerCompilerPass())
+            ->addCompilerPass(new BusCompilerPass())
+        ;
     }
 }

src/Command/AddScheduledDataflowCommand.php

@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace CodeRhapsodie\DataflowBundle\Command;
 
 use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
+use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
 use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface;
 use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
 use Symfony\Component\Console\Command\Command;
@@ -14,24 +15,16 @@ use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Style\SymfonyStyle;
 use Symfony\Component\Validator\Validator\ValidatorInterface;
 
+/**
+ * @codeCoverageIgnore
+ */
 class AddScheduledDataflowCommand extends Command
 {
     protected static $defaultName = 'code-rhapsodie:dataflow:schedule:add';
 
-    /** @var DataflowTypeRegistryInterface */
-    private $registry;
-    /** @var ScheduledDataflowRepository */
-    private $scheduledDataflowRepository;
-    /** @var ValidatorInterface */
-    private $validator;
-
-    public function __construct(DataflowTypeRegistryInterface $registry, ScheduledDataflowRepository $scheduledDataflowRepository, ValidatorInterface $validator)
+    public function __construct(private DataflowTypeRegistryInterface $registry, private ScheduledDataflowRepository $scheduledDataflowRepository, private ValidatorInterface $validator, private ConnectionFactory $connectionFactory)
     {
         parent::__construct();
-
-        $this->registry = $registry;
-        $this->scheduledDataflowRepository = $scheduledDataflowRepository;
-        $this->validator = $validator;
     }
 
     /**
@@ -44,17 +37,22 @@ class AddScheduledDataflowCommand extends Command
             ->setHelp('The <info>%command.name%</info> allows you to create a new scheduled dataflow.')
             ->addOption('label', null, InputOption::VALUE_REQUIRED, 'Label of the scheduled dataflow')
             ->addOption('type', null, InputOption::VALUE_REQUIRED, 'Type of the scheduled dataflow (FQCN)')
-            ->addOption('options', null, InputOption::VALUE_OPTIONAL, 'Options of the scheduled dataflow (ex: {"option1": "value1", "option2": "value2"})')
+            ->addOption('options', null, InputOption::VALUE_OPTIONAL,
+                'Options of the scheduled dataflow (ex: {"option1": "value1", "option2": "value2"})')
             ->addOption('frequency', null, InputOption::VALUE_REQUIRED, 'Frequency of the scheduled dataflow')
             ->addOption('first_run', null, InputOption::VALUE_REQUIRED, 'Date for the first run of the scheduled dataflow (Y-m-d H:i:s)')
-            ->addOption('enabled', null, InputOption::VALUE_REQUIRED, 'State of the scheduled dataflow');
+            ->addOption('enabled', null, InputOption::VALUE_REQUIRED, 'State of the scheduled dataflow')
+            ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
     }
 
     /**
      * {@inheritdoc}
      */
-    protected function execute(InputInterface $input, OutputInterface $output)
+    protected function execute(InputInterface $input, OutputInterface $output): int
     {
+        if (null !== $input->getOption('connection')) {
+            $this->connectionFactory->setConnectionName($input->getOption('connection'));
+        }
         $choices = [];
         $typeMapping = [];
         foreach ($this->registry->listDataflowTypes() as $fqcn => $dataflowType) {
@@ -74,11 +72,13 @@ class AddScheduledDataflowCommand extends Command
         }
         $options = $input->getOption('options');
         if (!$options) {
-            $options = $io->ask('What are the launch options for the scheduled dataflow? (ex: {"option1": "value1", "option2": "value2"})', json_encode([]));
+            $options = $io->ask('What are the launch options for the scheduled dataflow? (ex: {"option1": "value1", "option2": "value2"})',
+                json_encode([]));
         }
         $frequency = $input->getOption('frequency');
         if (!$frequency) {
-            $frequency = $io->choice('What is the frequency for the scheduled dataflow?', ScheduledDataflow::AVAILABLE_FREQUENCIES);
+            $frequency = $io->choice('What is the frequency for the scheduled dataflow?',
+                ScheduledDataflow::AVAILABLE_FREQUENCIES);
         }
         $firstRun = $input->getOption('first_run');
         if (!$firstRun) {
@@ -89,44 +89,27 @@ class AddScheduledDataflowCommand extends Command
             $enabled = $io->confirm('Enable the scheduled dataflow?');
         }
 
-        try {
-            $newScheduledDataflow = $this->createEntityFromArray([
-                'label' => $label,
-                'type' => $type,
-                'options' => $options,
-                'frequency' => $frequency,
-                'first_run' => $firstRun,
-                'enabled' => $enabled,
-            ]);
+        $newScheduledDataflow = ScheduledDataflow::createFromArray([
+            'id' => null,
+            'label' => $label,
+            'dataflow_type' => $type,
+            'options' => json_decode($options, true, 512, JSON_THROW_ON_ERROR),
+            'frequency' => $frequency,
+            'next' => new \DateTimeImmutable($firstRun),
+            'enabled' => $enabled,
+        ]);
 
-            $errors = $this->validator->validate($newScheduledDataflow);
-            if (count($errors) > 0) {
-                $io->error((string) $errors);
+        $errors = $this->validator->validate($newScheduledDataflow);
+        if (count($errors) > 0) {
+            $io->error((string) $errors);
 
-                return 2;
-            }
-
-            $this->scheduledDataflowRepository->save($newScheduledDataflow);
-            $io->success(sprintf('New scheduled dataflow "%s" (id:%d) was created successfully.', $newScheduledDataflow->getLabel(), $newScheduledDataflow->getId()));
-
-            return 0;
-        } catch (\Exception $e) {
-            $io->error(sprintf('An error occured when creating new scheduled dataflow : "%s".', $e->getMessage()));
-
-            return 1;
+            return 2;
         }
-    }
 
-    private function createEntityFromArray(array $input): ScheduledDataflow
-    {
-        $scheduledDataflow = new ScheduledDataflow();
-        $scheduledDataflow->setLabel($input['label']);
-        $scheduledDataflow->setDataflowType($input['type']);
-        $scheduledDataflow->setOptions(json_decode($input['options'], true));
-        $scheduledDataflow->setFrequency($input['frequency']);
-        $scheduledDataflow->setNext(new \DateTimeImmutable($input['first_run']));
-        $scheduledDataflow->setEnabled($input['enabled']);
+        $this->scheduledDataflowRepository->save($newScheduledDataflow);
+        $io->success(sprintf('New scheduled dataflow "%s" (id:%d) was created successfully.',
+            $newScheduledDataflow->getLabel(), $newScheduledDataflow->getId()));
 
-        return $scheduledDataflow;
+        return 0;
     }
 }

src/Command/ChangeScheduleStatusCommand.php

@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace CodeRhapsodie\DataflowBundle\Command;
 
 use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
+use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
 use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
@@ -13,18 +14,16 @@ use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Style\SymfonyStyle;
 
+/**
+ * @codeCoverageIgnore
+ */
 class ChangeScheduleStatusCommand extends Command
 {
     protected static $defaultName = 'code-rhapsodie:dataflow:schedule:change-status';
 
-    /** @var ScheduledDataflowRepository */
-    private $scheduledDataflowRepository;
-
-    public function __construct(ScheduledDataflowRepository $scheduledDataflowRepository)
+    public function __construct(private ScheduledDataflowRepository $scheduledDataflowRepository, private ConnectionFactory $connectionFactory)
     {
         parent::__construct();
-
-        $this->scheduledDataflowRepository = $scheduledDataflowRepository;
     }
 
     /**
@@ -37,17 +36,21 @@ class ChangeScheduleStatusCommand extends Command
             ->setHelp('The <info>%command.name%</info> command able you to change schedule status.')
             ->addArgument('schedule-id', InputArgument::REQUIRED, 'Id of the schedule')
             ->addOption('enable', null, InputOption::VALUE_NONE, 'Enable the schedule')
-            ->addOption('disable', null, InputOption::VALUE_NONE, 'Disable the schedule');
+            ->addOption('disable', null, InputOption::VALUE_NONE, 'Disable the schedule')
+            ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
     }
 
     /**
      * {@inheritdoc}
      */
-    protected function execute(InputInterface $input, OutputInterface $output)
+    protected function execute(InputInterface $input, OutputInterface $output): int
     {
+        if (null !== $input->getOption('connection')) {
+            $this->connectionFactory->setConnectionName($input->getOption('connection'));
+        }
         $io = new SymfonyStyle($input, $output);
         /** @var ScheduledDataflow|null $schedule */
-        $schedule = $this->scheduledDataflowRepository->find($input->getArgument('schedule-id'));
+        $schedule = $this->scheduledDataflowRepository->find((int) $input->getArgument('schedule-id'));
 
         if (!$schedule) {
             $io->error(sprintf('Cannot find scheduled dataflow with id "%d".', $input->getArgument('schedule-id')));

src/Command/ExecuteDataflowCommand.php

@@ -4,27 +4,31 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\Command;
 
+use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
 use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface;
+use Psr\Log\LoggerAwareInterface;
+use Psr\Log\LoggerAwareTrait;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
 
 /**
  * Runs one dataflow.
+ *
+ * @codeCoverageIgnore
  */
-class ExecuteDataflowCommand extends Command
+class ExecuteDataflowCommand extends Command implements LoggerAwareInterface
 {
+    use LoggerAwareTrait;
+
     protected static $defaultName = 'code-rhapsodie:dataflow:execute';
 
-    /** @var DataflowTypeRegistryInterface */
-    private $registry;
-
-    public function __construct(DataflowTypeRegistryInterface $registry)
+    public function __construct(private DataflowTypeRegistryInterface $registry, private ConnectionFactory $connectionFactory)
     {
         parent::__construct();
-
-        $this->registry = $registry;
     }
 
     /**
@@ -42,24 +46,38 @@ EOF
             )
             ->addArgument('fqcn', InputArgument::REQUIRED, 'FQCN or alias of the dataflow type')
             ->addArgument('options', InputArgument::OPTIONAL, 'Options for the dataflow type as a json string', '[]')
-        ;
+            ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
     }
 
     /**
      * {@inheritdoc}
      */
-    protected function execute(InputInterface $input, OutputInterface $output)
+    protected function execute(InputInterface $input, OutputInterface $output): int
     {
+        if (null !== $input->getOption('connection')) {
+            $this->connectionFactory->setConnectionName($input->getOption('connection'));
+        }
         $fqcnOrAlias = $input->getArgument('fqcn');
-        $options = json_decode($input->getArgument('options'), true);
+        $options = json_decode($input->getArgument('options'), true, 512, JSON_THROW_ON_ERROR);
+        $io = new SymfonyStyle($input, $output);
 
         $dataflowType = $this->registry->getDataflowType($fqcnOrAlias);
+        if ($dataflowType instanceof LoggerAwareInterface && isset($this->logger)) {
+            $dataflowType->setLogger($this->logger);
+        }
+
         $result = $dataflowType->process($options);
 
-        $output->writeln('Executed: '.$result->getName());
-        $output->writeln('Start time: '.$result->getStartTime()->format('Y/m/d H:i:s'));
-        $output->writeln('End time: '.$result->getEndTime()->format('Y/m/d H:i:s'));
-        $output->writeln('Success: '.$result->getSuccessCount());
+        $io->writeln('Executed: '.$result->getName());
+        $io->writeln('Start time: '.$result->getStartTime()->format('Y/m/d H:i:s'));
+        $io->writeln('End time: '.$result->getEndTime()->format('Y/m/d H:i:s'));
+        $io->writeln('Success: '.$result->getSuccessCount());
+
+        if ($result->hasErrors()) {
+            $io->error("Errors: {$result->getErrorCount()}\nExceptions traces are available in the logs.");
+
+            return 1;
+        }
 
         return 0;
     }

src/Command/JobShowCommand.php

@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace CodeRhapsodie\DataflowBundle\Command;
 
 use CodeRhapsodie\DataflowBundle\Entity\Job;
+use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
 use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
@@ -12,6 +13,9 @@ use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Style\SymfonyStyle;
 
+/**
+ * @codeCoverageIgnore
+ */
 class JobShowCommand extends Command
 {
     private const STATUS_MAPPING = [
@@ -22,14 +26,9 @@ class JobShowCommand extends Command
 
     protected static $defaultName = 'code-rhapsodie:dataflow:job:show';
 
-    /** @var JobRepository */
-    private $jobRepository;
-
-    public function __construct(JobRepository $jobRepository)
+    public function __construct(private JobRepository $jobRepository, private ConnectionFactory $connectionFactory)
     {
         parent::__construct();
-
-        $this->jobRepository = $jobRepository;
     }
 
     /**
@@ -42,14 +41,19 @@ class JobShowCommand extends Command
             ->setHelp('The <info>%command.name%</info> display job details for schedule or specific job.')
             ->addOption('job-id', null, InputOption::VALUE_REQUIRED, 'Id of the job to get details')
             ->addOption('schedule-id', null, InputOption::VALUE_REQUIRED, 'Id of schedule for last execution details')
-            ->addOption('details', null, InputOption::VALUE_NONE, 'Display full details');
+            ->addOption('details', null, InputOption::VALUE_NONE, 'Display full details')
+            ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
     }
 
     /**
      * {@inheritdoc}
      */
-    protected function execute(InputInterface $input, OutputInterface $output)
+    protected function execute(InputInterface $input, OutputInterface $output): int
     {
+        if (null !== $input->getOption('connection')) {
+            $this->connectionFactory->setConnectionName($input->getOption('connection'));
+        }
+
         $io = new SymfonyStyle($input, $output);
 
         $jobId = (int) $input->getOption('job-id');
@@ -84,21 +88,19 @@ class JobShowCommand extends Command
             ['Started at', $job->getStartTime() ? $job->getStartTime()->format('Y-m-d H:i:s') : '-'],
             ['Ended at', $job->getEndTime() ? $job->getEndTime()->format('Y-m-d H:i:s') : '-'],
             ['Object number', $job->getCount()],
-            ['Errors', count($job->getExceptions())],
+            ['Errors', count((array) $job->getExceptions())],
             ['Status', $this->translateStatus($job->getStatus())],
         ];
         if ($input->getOption('details')) {
             $display[] = ['Type', $job->getDataflowType()];
-            $display[] = ['Options', json_encode($job->getOptions())];
+            $display[] = ['Options', json_encode($job->getOptions(), JSON_THROW_ON_ERROR)];
             $io->section('Summary');
         }
 
         $io->table(['Field', 'Value'], $display);
         if ($input->getOption('details')) {
             $io->section('Exceptions');
-            $exceptions = array_map(function (string $exception) {
-                return substr($exception, 0, 900).'…';
-            }, $job->getExceptions());
+            $exceptions = array_map(fn(string $exception) => substr($exception, 0, 900).'…', $job->getExceptions());
 
             $io->write($exceptions);
         }

src/Command/RunPendingDataflowsCommand.php

@@ -4,15 +4,19 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\Command;
 
+use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
 use CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManagerInterface;
 use CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunnerInterface;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Command\LockableTrait;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 /**
  * Runs dataflows according to user-defined schedule.
+ *
+ * @codeCoverageIgnore
  */
 class RunPendingDataflowsCommand extends Command
 {
@@ -20,18 +24,9 @@ class RunPendingDataflowsCommand extends Command
 
     protected static $defaultName = 'code-rhapsodie:dataflow:run-pending';
 
-    /** @var ScheduledDataflowManagerInterface */
-    private $manager;
-
-    /** @var PendingDataflowRunnerInterface */
-    private $runner;
-
-    public function __construct(ScheduledDataflowManagerInterface $manager, PendingDataflowRunnerInterface $runner)
+    public function __construct(private ScheduledDataflowManagerInterface $manager, private PendingDataflowRunnerInterface $runner, private ConnectionFactory $connectionFactory)
     {
         parent::__construct();
-
-        $this->manager = $manager;
-        $this->runner = $runner;
     }
 
     /**
@@ -45,13 +40,13 @@ class RunPendingDataflowsCommand extends Command
 The <info>%command.name%</info> command runs dataflows according to the schedule defined in the UI by the user.
 EOF
             )
-        ;
+            ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
     }
 
     /**
      * {@inheritdoc}
      */
-    protected function execute(InputInterface $input, OutputInterface $output)
+    protected function execute(InputInterface $input, OutputInterface $output): int
     {
         if (!$this->lock()) {
             $output->writeln('The command is already running in another process.');
@@ -59,6 +54,10 @@ EOF
             return 0;
         }
 
+        if (null !== $input->getOption('connection')) {
+            $this->connectionFactory->setConnectionName($input->getOption('connection'));
+        }
+
         $this->manager->createJobsFromScheduledDataflows();
         $this->runner->runPendingDataflows();
 

src/Command/ScheduleListCommand.php

@@ -4,24 +4,24 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\Command;
 
+use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
 use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Style\SymfonyStyle;
 
+/**
+ * @codeCoverageIgnore
+ */
 class ScheduleListCommand extends Command
 {
     protected static $defaultName = 'code-rhapsodie:dataflow:schedule:list';
 
-    /** @var ScheduledDataflowRepository */
-    private $scheduledDataflowRepository;
-
-    public function __construct(ScheduledDataflowRepository $scheduledDataflowRepository)
+    public function __construct(private ScheduledDataflowRepository $scheduledDataflowRepository, private ConnectionFactory $connectionFactory)
     {
         parent::__construct();
-
-        $this->scheduledDataflowRepository = $scheduledDataflowRepository;
     }
 
     /**
@@ -31,14 +31,18 @@ class ScheduleListCommand extends Command
     {
         $this
             ->setDescription('List scheduled dataflows')
-            ->setHelp('The <info>%command.name%</info> lists all scheduled dataflows.');
+            ->setHelp('The <info>%command.name%</info> lists all scheduled dataflows.')
+            ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
     }
 
     /**
      * {@inheritdoc}
      */
-    protected function execute(InputInterface $input, OutputInterface $output)
+    protected function execute(InputInterface $input, OutputInterface $output): int
     {
+        if (null !== $input->getOption('connection')) {
+            $this->connectionFactory->setConnectionName($input->getOption('connection'));
+        }
         $io = new SymfonyStyle($input, $output);
         $display = [];
         $schedules = $this->scheduledDataflowRepository->listAllOrderedByLabel();
@@ -48,7 +52,7 @@ class ScheduleListCommand extends Command
                 $schedule['label'],
                 $schedule['enabled'] ? 'yes' : 'no',
                 $schedule['startTime'] ? (new \DateTime($schedule['startTime']))->format('Y-m-d H:i:s') : '-',
-                $schedule['next'] ? $schedule['next']->format('Y-m-d H:i:s') : '-',
+                $schedule['next'] ? (new \DateTime($schedule['next']))->format('Y-m-d H:i:s') : '-',
             ];
         }
 

src/Command/SchemaCommand.php

@@ -0,0 +1,96 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Command;
+
+use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
+use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
+use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
+use CodeRhapsodie\DataflowBundle\SchemaProvider\DataflowSchemaProvider;
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Schema\Table;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+
+/**
+ * @codeCoverageIgnore
+ */
+class SchemaCommand extends Command
+{
+    protected static $defaultName = 'code-rhapsodie:dataflow:dump-schema';
+
+    public function __construct(private ConnectionFactory $connectionFactory)
+    {
+        parent::__construct();
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function configure()
+    {
+        $this
+            ->setDescription('Generates schema create / update SQL queries')
+            ->setHelp('The <info>%command.name%</info> help you to generate SQL Query to create or update your database schema for this bundle')
+            ->addOption('update', null, InputOption::VALUE_NONE, 'Dump only the update SQL queries.')
+            ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use')
+        ;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        if (null !== $input->getOption('connection')) {
+            $this->connectionFactory->setConnectionName($input->getOption('connection'));
+        }
+
+        $connection = $this->connectionFactory->getConnection();
+
+        $schemaProvider = new DataflowSchemaProvider();
+        $schema = $schemaProvider->createSchema();
+
+        $sqls = $schema->toSql($connection->getDatabasePlatform());
+
+        if ($input->getOption('update')) {
+            $sm = $connection->getSchemaManager();
+
+            $tableArray = [JobRepository::TABLE_NAME, ScheduledDataflowRepository::TABLE_NAME];
+            $tables = [];
+            foreach ($sm->listTables() as $table) {
+                /** @var Table $table */
+                if (in_array($table->getName(), $tableArray)) {
+                    $tables[] = $table;
+                }
+            }
+
+            $namespaces = [];
+
+            if ($connection->getDatabasePlatform()->supportsSchemas()) {
+                $namespaces = $sm->listNamespaceNames();
+            }
+
+            $sequences = [];
+
+            if ($connection->getDatabasePlatform()->supportsSequences()) {
+                $sequences = $sm->listSequences();
+            }
+
+            $oldSchema = new Schema($tables, $sequences, $sm->createSchemaConfig(), $namespaces);
+
+            $sqls = $schema->getMigrateFromSql($oldSchema, $connection->getDatabasePlatform());
+        }
+        $io = new SymfonyStyle($input, $output);
+        $io->text('Execute these SQL Queries on your database:');
+        foreach ($sqls as $sql) {
+            $io->text($sql.';');
+        }
+
+        return 0;
+    }
+}

src/DataflowType/AMPAsyncDataflowBuilder.php

@@ -0,0 +1,72 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\DataflowType;
+
+use CodeRhapsodie\DataflowBundle\DataflowType\Dataflow\AMPAsyncDataflow;
+use CodeRhapsodie\DataflowBundle\DataflowType\Dataflow\DataflowInterface;
+use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
+use Symfony\Component\OptionsResolver\OptionsResolver;
+
+class AMPAsyncDataflowBuilder extends DataflowBuilder
+{
+    public function __construct(protected ?int $loopInterval = 0, protected ?int $emitInterval = 0)
+    {
+    }
+
+    private ?string $name = null;
+
+    private ?iterable $reader = null;
+
+    private array $steps = [];
+
+    /** @var WriterInterface[] */
+    private array $writers = [];
+
+    public function setName(string $name): self
+    {
+        $this->name = $name;
+
+        return $this;
+    }
+
+    public function setReader(iterable $reader): self
+    {
+        $this->reader = $reader;
+
+        return $this;
+    }
+
+    public function addStep(callable $step, int $priority = 0, int $scale = 1): self
+    {
+        $this->steps[$priority][] = ['step' => $step, 'scale' => $scale];
+
+        return $this;
+    }
+
+    public function addWriter(WriterInterface $writer): self
+    {
+        $this->writers[] = $writer;
+
+        return $this;
+    }
+
+    public function getDataflow(): DataflowInterface
+    {
+        $dataflow = new AMPAsyncDataflow($this->reader, $this->name, $this->loopInterval, $this->emitInterval);
+
+        krsort($this->steps);
+        foreach ($this->steps as $stepArray) {
+            foreach ($stepArray as $step) {
+                $dataflow->addStep($step['step'], $step['scale']);
+            }
+        }
+
+        foreach ($this->writers as $writer) {
+            $dataflow->addWriter($writer);
+        }
+
+        return $dataflow;
+    }
+}

src/DataflowType/AbstractDataflowType.php

@@ -4,10 +4,18 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\DataflowType;
 
+use Psr\Log\LoggerAwareInterface;
+use Psr\Log\LoggerAwareTrait;
+use Psr\Log\LoggerInterface;
 use Symfony\Component\OptionsResolver\OptionsResolver;
 
-abstract class AbstractDataflowType implements DataflowTypeInterface
+abstract class AbstractDataflowType implements DataflowTypeInterface, LoggerAwareInterface
 {
+    use LoggerAwareTrait;
+
+    /**
+     * @codeCoverageIgnore
+     */
     public function getAliases(): iterable
     {
         return [];
@@ -19,15 +27,25 @@ abstract class AbstractDataflowType implements DataflowTypeInterface
         $this->configureOptions($optionsResolver);
         $options = $optionsResolver->resolve($options);
 
-        $builder = (new DataflowBuilder())
-            ->setName($this->getLabel())
-        ;
+        $builder = $this->createDataflowBuilder();
+        $builder->setName($this->getLabel());
         $this->buildDataflow($builder, $options);
         $dataflow = $builder->getDataflow();
+        if ($dataflow instanceof LoggerAwareInterface && $this->logger instanceof LoggerInterface) {
+            $dataflow->setLogger($this->logger);
+        }
 
         return $dataflow->process();
     }
 
+    protected function createDataflowBuilder(): DataflowBuilder
+    {
+        return new DataflowBuilder();
+    }
+
+    /**
+     * @codeCoverageIgnore
+     */
     protected function configureOptions(OptionsResolver $optionsResolver): void
     {
     }

src/DataflowType/Dataflow/AMPAsyncDataflow.php

@@ -0,0 +1,164 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\DataflowType\Dataflow;
+
+use function Amp\coroutine;
+use Amp\Deferred;
+use Amp\Delayed;
+use Amp\Loop;
+use Amp\Producer;
+use Amp\Promise;
+use function Amp\Promise\wait;
+use CodeRhapsodie\DataflowBundle\DataflowType\Result;
+use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
+use Psr\Log\LoggerAwareInterface;
+use Psr\Log\LoggerAwareTrait;
+use RuntimeException;
+use Throwable;
+
+class AMPAsyncDataflow implements DataflowInterface, LoggerAwareInterface
+{
+    use LoggerAwareTrait;
+
+    /** @var callable[] */
+    private array $steps = [];
+
+    /** @var WriterInterface[] */
+    private array $writers = [];
+
+    private array $states = [];
+
+    private array $stepsJobs = [];
+
+    public function __construct(private iterable $reader, private ?string $name, private ?int $loopInterval = 0, private ?int $emitInterval = 0)
+    {
+        if (!function_exists('Amp\\Promise\\wait')) {
+            throw new RuntimeException('Amp is not loaded. Suggest install it with composer require amphp/amp');
+        }
+    }
+
+    /**
+     * @param int $scale
+     *
+     * @return $this
+     */
+    public function addStep(callable $step, $scale = 1): self
+    {
+        $this->steps[] = [$step, $scale];
+
+        return $this;
+    }
+
+    /**
+     * @return $this
+     */
+    public function addWriter(WriterInterface $writer): self
+    {
+        $this->writers[] = $writer;
+
+        return $this;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function process(): Result
+    {
+        $count = 0;
+        $exceptions = [];
+        $startTime = new \DateTimeImmutable();
+
+        try {
+            foreach ($this->writers as $writer) {
+                $writer->prepare();
+            }
+
+            $deferred = new Deferred();
+            $resolved = false; //missing $deferred->isResolved() in version 2.5
+            $producer = new Producer(function (callable $emit) {
+                foreach ($this->reader as $index => $item) {
+                    yield new Delayed($this->emitInterval);
+                    yield $emit([$index, $item]);
+                }
+            });
+
+            $watcherId = Loop::repeat($this->loopInterval, function () use ($deferred, &$resolved, $producer, &$count, &$exceptions) {
+                if (yield $producer->advance()) {
+                    $it = $producer->getCurrent();
+                    [$index, $item] = $it;
+                    $this->states[$index] = [$index, 0, $item];
+                } elseif (!$resolved && 0 === count($this->states)) {
+                    $resolved = true;
+                    $deferred->resolve();
+                }
+
+                foreach ($this->states as $state) {
+                    $this->processState($state, $count, $exceptions);
+                }
+            });
+
+            wait($deferred->promise());
+            Loop::cancel($watcherId);
+
+            foreach ($this->writers as $writer) {
+                $writer->finish();
+            }
+        } catch (\Throwable $e) {
+            $exceptions[] = $e;
+            $this->logException($e);
+        }
+
+        return new Result($this->name, $startTime, new \DateTimeImmutable(), $count, $exceptions);
+    }
+
+    /**
+     * @param int   $count      internal count reference
+     * @param array $exceptions internal exceptions
+     */
+    private function processState(mixed $state, int &$count, array &$exceptions): void
+    {
+        [$readIndex, $stepIndex, $item] = $state;
+        if ($stepIndex < count($this->steps)) {
+            if (!isset($this->stepsJobs[$stepIndex])) {
+                $this->stepsJobs[$stepIndex] = [];
+            }
+            [$step, $scale] = $this->steps[$stepIndex];
+            if ((is_countable($this->stepsJobs[$stepIndex]) ? count($this->stepsJobs[$stepIndex]) : 0) < $scale && !isset($this->stepsJobs[$stepIndex][$readIndex])) {
+                $this->stepsJobs[$stepIndex][$readIndex] = true;
+                /** @var Promise<void> $promise */
+                $promise = coroutine($step)($item);
+                $promise->onResolve(function (?Throwable $exception = null, $newItem = null) use ($stepIndex, $readIndex, &$exceptions) {
+                    if ($exception) {
+                        $exceptions[$stepIndex] = $exception;
+                        $this->logException($exception, (string) $stepIndex);
+                    } elseif (false === $newItem) {
+                        unset($this->states[$readIndex]);
+                    } else {
+                        $this->states[$readIndex] = [$readIndex, $stepIndex + 1, $newItem];
+                    }
+
+                    unset($this->stepsJobs[$stepIndex][$readIndex]);
+                });
+            }
+        } else {
+            unset($this->states[$readIndex]);
+
+            foreach ($this->writers as $writer) {
+                $writer->write($item);
+            }
+
+            ++$count;
+        }
+    }
+
+    private function logException(Throwable $e, ?string $index = null): void
+    {
+        if (!isset($this->logger)) {
+            return;
+        }
+
+        $this->logger->error($e, ['exception' => $e, 'index' => $index]);
+    }
+}

src/DataflowType/Dataflow/Dataflow.php

@@ -6,36 +6,26 @@ namespace CodeRhapsodie\DataflowBundle\DataflowType\Dataflow;
 
 use CodeRhapsodie\DataflowBundle\DataflowType\Result;
 use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
-use CodeRhapsodie\DataflowBundle\Exceptions\InterruptedProcessingException;
-use Seld\Signal\SignalHandler;
+use Psr\Log\LoggerAwareInterface;
+use Psr\Log\LoggerAwareTrait;
 
-class Dataflow implements DataflowInterface
+class Dataflow implements DataflowInterface, LoggerAwareInterface
 {
-    /** @var string */
-    private $name;
-
-    /** @var iterable */
-    private $reader;
+    use LoggerAwareTrait;
 
     /** @var callable[] */
-    private $steps = [];
+    private array $steps = [];
 
     /** @var WriterInterface[] */
-    private $writers = [];
+    private array $writers = [];
 
-    /**
-     * @param iterable    $reader
-     * @param null|string $name
-     */
-    public function __construct(iterable $reader, ?string $name)
+    private ?\Closure $customExceptionIndex = null;
+
+    public function __construct(private iterable $reader, private ?string $name)
     {
-        $this->reader = $reader;
-        $this->name = $name;
     }
 
     /**
-     * @param callable $step
-     *
      * @return $this
      */
     public function addStep(callable $step): self
@@ -46,8 +36,6 @@ class Dataflow implements DataflowInterface
     }
 
     /**
-     * @param WriterInterface $writer
-     *
      * @return $this
      */
     public function addWriter(WriterInterface $writer): self
@@ -57,44 +45,62 @@ class Dataflow implements DataflowInterface
         return $this;
     }
 
+    /**
+     * @return $this
+     */
+    public function setCustomExceptionIndex(callable $callable): self
+    {
+        $this->customExceptionIndex = \Closure::fromCallable($callable);
+
+        return $this;
+    }
+
     /**
      * {@inheritdoc}
      */
     public function process(): Result
     {
         $count = 0;
-        $exceptions = new \SplObjectStorage();
-        $startTime = new \DateTime();
+        $exceptions = [];
+        $startTime = new \DateTimeImmutable();
 
-        SignalHandler::create(['SIGTERM', 'SIGINT'], function () {
-            throw new InterruptedProcessingException();
-        });
-
-        foreach ($this->writers as $writer) {
-            $writer->prepare();
-        }
-
-        foreach ($this->reader as $index => $item) {
-            try {
-                $this->processItem($item);
-            } catch (\Exception $e) {
-                $exceptions->attach($e, $index);
+        try {
+            foreach ($this->writers as $writer) {
+                $writer->prepare();
             }
 
-            ++$count;
+            foreach ($this->reader as $index => $item) {
+                try {
+                    $this->processItem($item);
+                } catch (\Throwable $e) {
+                    $exceptionIndex = $index;
+                    try {
+                        if (is_callable($this->customExceptionIndex)) {
+                            $exceptionIndex = (string) ($this->customExceptionIndex)($item, $index);
+                        }
+                    } catch (\Throwable $e2) {
+                        $exceptions[$index] = $e2;
+                        $this->logException($e2, $index);
+                    }
+                    $exceptions[$exceptionIndex] = $e;
+                    $this->logException($e, $exceptionIndex);
+                }
+
+                ++$count;
+            }
+
+            foreach ($this->writers as $writer) {
+                $writer->finish();
+            }
+        } catch (\Throwable $e) {
+            $exceptions[] = $e;
+            $this->logException($e);
         }
 
-        foreach ($this->writers as $writer) {
-            $writer->finish();
-        }
-
-        return new Result($this->name, $startTime, new \DateTime(), $count, $exceptions);
+        return new Result($this->name, $startTime, new \DateTimeImmutable(), $count, $exceptions);
     }
 
-    /**
-     * @param mixed $item
-     */
-    private function processItem($item): void
+    private function processItem(mixed $item): void
     {
         foreach ($this->steps as $step) {
             $item = call_user_func($step, $item);
@@ -108,4 +114,13 @@ class Dataflow implements DataflowInterface
             $writer->write($item);
         }
     }
+
+    private function logException(\Throwable $e, string|int|null $index = null): void
+    {
+        if (!isset($this->logger)) {
+            return;
+        }
+
+        $this->logger->error($e, ['exception' => $e, 'index' => $index]);
+    }
 }

src/DataflowType/Dataflow/DataflowInterface.php

@@ -13,8 +13,6 @@ interface DataflowInterface
 {
     /**
      * Processes the data.
-     *
-     * @return Result
      */
     public function process(): Result;
 }

src/DataflowType/DataflowBuilder.php

@@ -10,17 +10,16 @@ use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
 
 class DataflowBuilder
 {
-    /** @var string */
-    private $name;
+    private ?string $name = null;
 
-    /** @var iterable */
-    private $reader;
+    private ?iterable $reader = null;
 
-    /** @var array */
-    private $steps = [];
+    private array $steps = [];
 
     /** @var WriterInterface[] */
-    private $writers = [];
+    private array $writers = [];
+
+    private ?\Closure $customExceptionIndex = null;
 
     public function setName(string $name): self
     {
@@ -50,6 +49,13 @@ class DataflowBuilder
         return $this;
     }
 
+    public function setCustomExceptionIndex(callable $callable): self
+    {
+        $this->customExceptionIndex = \Closure::fromCallable($callable);
+
+        return $this;
+    }
+
     public function getDataflow(): DataflowInterface
     {
         $dataflow = new Dataflow($this->reader, $this->name);
@@ -65,6 +71,10 @@ class DataflowBuilder
             $dataflow->addWriter($writer);
         }
 
+        if (is_callable($this->customExceptionIndex)) {
+            $dataflow->setCustomExceptionIndex($this->customExceptionIndex);
+        }
+
         return $dataflow;
     }
 }

src/DataflowType/Result.php

@@ -4,119 +4,68 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\DataflowType;
 
+/**
+ * @codeCoverageIgnore
+ */
 class Result
 {
-    /** @var string */
-    private $name;
+    private \DateInterval $elapsed;
 
-    /** @var \DateTimeInterface */
-    private $startTime;
+    private int $errorCount = 0;
 
-    /** @var \DateTimeInterface */
-    private $endTime;
+    private int $successCount = 0;
 
-    /** @var \DateInterval */
-    private $elapsed;
+    private array $exceptions;
 
-    /** @var int */
-    private $errorCount = 0;
-
-    /** @var int */
-    private $successCount = 0;
-
-    /** @var int */
-    private $totalProcessedCount = 0;
-
-    /** @var \SplObjectStorage */
-    private $exceptions;
-
-    /**
-     * @param string             $name
-     * @param \DateTimeInterface $startTime
-     * @param \DateTimeInterface $endTime
-     * @param int                $totalCount
-     * @param \SplObjectStorage  $exceptions
-     */
-    public function __construct(string $name, \DateTimeInterface $startTime, \DateTimeInterface $endTime, int $totalCount, \SplObjectStorage $exceptions)
+    public function __construct(private string $name, private \DateTimeInterface $startTime, private \DateTimeInterface $endTime, private int $totalProcessedCount, array $exceptions)
     {
-        $this->name = $name;
-        $this->startTime = $startTime;
-        $this->endTime = $endTime;
         $this->elapsed = $startTime->diff($endTime);
-        $this->totalProcessedCount = $totalCount;
         $this->errorCount = count($exceptions);
-        $this->successCount = $totalCount - $this->errorCount;
+        $this->successCount = $totalProcessedCount - $this->errorCount;
         $this->exceptions = $exceptions;
     }
 
-    /**
-     * @return string
-     */
     public function getName(): string
     {
         return $this->name;
     }
 
-    /**
-     * @return \DateTimeInterface
-     */
     public function getStartTime(): \DateTimeInterface
     {
         return $this->startTime;
     }
 
-    /**
-     * @return \DateTimeInterface
-     */
     public function getEndTime(): \DateTimeInterface
     {
         return $this->endTime;
     }
 
-    /**
-     * @return \DateInterval
-     */
     public function getElapsed(): \DateInterval
     {
         return $this->elapsed;
     }
 
-    /**
-     * @return int
-     */
     public function getErrorCount(): int
     {
         return $this->errorCount;
     }
 
-    /**
-     * @return int
-     */
     public function getSuccessCount(): int
     {
         return $this->successCount;
     }
 
-    /**
-     * @return int
-     */
     public function getTotalProcessedCount(): int
     {
         return $this->totalProcessedCount;
     }
 
-    /**
-     * @return bool
-     */
     public function hasErrors(): bool
     {
         return $this->errorCount > 0;
     }
 
-    /**
-     * @return \SplObjectStorage
-     */
-    public function getExceptions(): \SplObjectStorage
+    public function getExceptions(): array
     {
         return $this->exceptions;
     }

src/DataflowType/Writer/CollectionWriter.php

@@ -0,0 +1,58 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\DataflowType\Writer;
+
+use CodeRhapsodie\DataflowBundle\Exceptions\UnsupportedItemTypeException;
+
+/**
+ * Delegates the writing of each item in a collection to an embedded writer.
+ */
+class CollectionWriter implements DelegateWriterInterface
+{
+    /**
+     * CollectionWriter constructor.
+     */
+    public function __construct(private WriterInterface $writer)
+    {
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function prepare()
+    {
+        $this->writer->prepare();
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function write($collection)
+    {
+        if (!is_iterable($collection)) {
+            throw new UnsupportedItemTypeException(sprintf('Item to write was expected to be an iterable, received %s.', get_debug_type($collection)));
+        }
+
+        foreach ($collection as $item) {
+            $this->writer->write($item);
+        }
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function finish()
+    {
+        $this->writer->finish();
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function supports($item): bool
+    {
+        return is_iterable($item);
+    }
+}

src/DataflowType/Writer/DelegateWriterInterface.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\DataflowType\Writer;
+
+/**
+ * A writer that can be used as a delegate of DelegatorWriter.
+ */
+interface DelegateWriterInterface extends WriterInterface
+{
+    /**
+     * Returns true if the argument is of a supported type.
+     *
+     * @param $item
+     */
+    public function supports($item): bool;
+}

src/DataflowType/Writer/DelegatorWriter.php

@@ -0,0 +1,95 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\DataflowType\Writer;
+
+use CodeRhapsodie\DataflowBundle\Exceptions\UnsupportedItemTypeException;
+
+/**
+ * Writer that delegated the actual writing to other writers.
+ */
+class DelegatorWriter implements DelegateWriterInterface
+{
+    /** @var DelegateWriterInterface[] */
+    private array $delegates = [];
+
+    /**
+     * DelegatorWriter constructor.
+     */
+    public function __construct()
+    {
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function prepare()
+    {
+        foreach ($this->delegates as $delegate) {
+            $delegate->prepare();
+        }
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function write($item)
+    {
+        foreach ($this->delegates as $delegate) {
+            if (!$delegate->supports($item)) {
+                continue;
+            }
+
+            $delegate->write($item);
+
+            return;
+        }
+
+        throw new UnsupportedItemTypeException(sprintf('None of the registered delegate writers support the received item of type %s', get_debug_type($item)));
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function finish()
+    {
+        foreach ($this->delegates as $delegate) {
+            $delegate->finish();
+        }
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function supports($item): bool
+    {
+        foreach ($this->delegates as $delegate) {
+            if ($delegate->supports($item)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Registers a collection of delegates.
+     *
+     * @param iterable|DelegateWriterInterface[] $delegates
+     */
+    public function addDelegates(iterable $delegates): void
+    {
+        foreach ($delegates as $delegate) {
+            $this->addDelegate($delegate);
+        }
+    }
+
+    /**
+     * Registers one delegate.
+     */
+    public function addDelegate(DelegateWriterInterface $delegate): void
+    {
+        $this->delegates[] = $delegate;
+    }
+}

src/DataflowType/Writer/PortWriterAdapter.php

@@ -6,12 +6,8 @@ namespace CodeRhapsodie\DataflowBundle\DataflowType\Writer;
 
 class PortWriterAdapter implements WriterInterface
 {
-    /** @var \Port\Writer */
-    private $writer;
-
-    public function __construct(\Port\Writer $writer)
+    public function __construct(private \Port\Writer $writer)
     {
-        $this->writer = $writer;
     }
 
     public function prepare()

src/DataflowType/Writer/WriterInterface.php

@@ -4,11 +4,23 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\DataflowType\Writer;
 
+/**
+ * Represents a writer for dataflows.
+ */
 interface WriterInterface
 {
+    /**
+     * Called before the dataflow is processed.
+     */
     public function prepare();
 
-    public function write($item);
+    /**
+     * Write an item.
+     */
+    public function write(mixed $item);
 
+    /**
+     * Called after the dataflow is processed.
+     */
     public function finish();
 }

src/DependencyInjection/CodeRhapsodieDataflowExtension.php

@@ -10,6 +10,9 @@ use Symfony\Component\DependencyInjection\ContainerBuilder;
 use Symfony\Component\DependencyInjection\Extension\Extension;
 use Symfony\Component\DependencyInjection\Loader\YamlFileLoader;
 
+/**
+ * @codeCoverageIgnore
+ */
 class CodeRhapsodieDataflowExtension extends Extension
 {
     public function load(array $configs, ContainerBuilder $container)
@@ -21,5 +24,15 @@ class CodeRhapsodieDataflowExtension extends Extension
             ->registerForAutoconfiguration(DataflowTypeInterface::class)
             ->addTag('coderhapsodie.dataflow.type')
         ;
+        $configuration = new Configuration();
+        $config = $this->processConfiguration($configuration, $configs);
+
+        $container->setParameter('coderhapsodie.dataflow.dbal_default_connection', $config['dbal_default_connection']);
+        $container->setParameter('coderhapsodie.dataflow.default_logger', $config['default_logger']);
+
+        if ($config['messenger_mode']['enabled']) {
+            $container->setParameter('coderhapsodie.dataflow.bus', $config['messenger_mode']['bus']);
+            $loader->load('messenger_services.yaml');
+        }
     }
 }

src/DependencyInjection/Compiler/BusCompilerPass.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler;
+
+use CodeRhapsodie\DataflowBundle\Runner\MessengerDataflowRunner;
+use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
+use Symfony\Component\DependencyInjection\ContainerBuilder;
+use Symfony\Component\DependencyInjection\Exception\InvalidArgumentException;
+use Symfony\Component\DependencyInjection\Reference;
+
+class BusCompilerPass implements CompilerPassInterface
+{
+    /**
+     * {@inheritdoc}
+     */
+    public function process(ContainerBuilder $container)
+    {
+        if (!$container->hasParameter('coderhapsodie.dataflow.bus')) {
+            return;
+        }
+
+        $bus = $container->getParameter('coderhapsodie.dataflow.bus');
+        if (!$container->has($bus)) {
+            throw new InvalidArgumentException(sprintf('Service "%s" not found', $bus));
+        }
+
+        if (!$container->has(MessengerDataflowRunner::class)) {
+            return;
+        }
+
+        $definition = $container->findDefinition(MessengerDataflowRunner::class);
+        $definition->setArgument('$bus', new Reference($bus));
+    }
+}

src/DependencyInjection/Compiler/DataflowTypeCompilerPass.php

@@ -11,6 +11,8 @@ use Symfony\Component\DependencyInjection\Reference;
 
 /**
  * Registers dataflow types in the registry.
+ *
+ * @codeCoverageIgnore
  */
 class DataflowTypeCompilerPass implements CompilerPassInterface
 {

src/DependencyInjection/Compiler/DefaultLoggerCompilerPass.php

@@ -0,0 +1,34 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler;
+
+use CodeRhapsodie\DataflowBundle\Command\ExecuteDataflowCommand;
+use CodeRhapsodie\DataflowBundle\Processor\JobProcessor;
+use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
+use Symfony\Component\DependencyInjection\ContainerBuilder;
+use Symfony\Component\DependencyInjection\Reference;
+
+class DefaultLoggerCompilerPass implements CompilerPassInterface
+{
+    /**
+     * {@inheritdoc}
+     */
+    public function process(ContainerBuilder $container)
+    {
+        $defaultLogger = $container->getParameter('coderhapsodie.dataflow.default_logger');
+        if (!$container->has($defaultLogger)) {
+            return;
+        }
+
+        foreach ([ExecuteDataflowCommand::class, JobProcessor::class] as $serviceId) {
+            if (!$container->has($serviceId)) {
+                continue;
+            }
+
+            $definition = $container->findDefinition($serviceId);
+            $definition->addMethodCall('setLogger', [new Reference($defaultLogger)]);
+        }
+    }
+}

src/DependencyInjection/Configuration.php

@@ -0,0 +1,51 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\DependencyInjection;
+
+use Symfony\Component\Config\Definition\Builder\TreeBuilder;
+use Symfony\Component\Config\Definition\ConfigurationInterface;
+use Symfony\Component\Messenger\MessageBusInterface;
+
+class Configuration implements ConfigurationInterface
+{
+    public function getConfigTreeBuilder(): \Symfony\Component\Config\Definition\Builder\TreeBuilder
+    {
+        $treeBuilder = new TreeBuilder('code_rhapsodie_dataflow');
+        if (method_exists($treeBuilder, 'getRootNode')) {
+            $rootNode = $treeBuilder->getRootNode();
+        } else {
+            // BC for symfony/config < 4.2
+            $rootNode = $treeBuilder->root('code_rhapsodie_dataflow');
+        }
+
+        $rootNode
+            ->children()
+                ->scalarNode('dbal_default_connection')
+                    ->defaultValue('default')
+                ->end()
+                ->scalarNode('default_logger')
+                    ->defaultValue('logger')
+                ->end()
+                ->arrayNode('messenger_mode')
+                    ->addDefaultsIfNotSet()
+                    ->children()
+                        ->booleanNode('enabled')
+                            ->defaultFalse()
+                        ->end()
+                        ->scalarNode('bus')
+                            ->defaultValue('messenger.default_bus')
+                        ->end()
+                    ->end()
+                    ->validate()
+                        ->ifTrue(static fn($v): bool => $v['enabled'] && !interface_exists(MessageBusInterface::class))
+                        ->thenInvalid('You need "symfony/messenger" in order to use Dataflow messenger mode.')
+                    ->end()
+                ->end()
+            ->end()
+        ;
+
+        return $treeBuilder;
+    }
+}

src/Entity/Job.php

@@ -4,105 +4,63 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\Entity;
 
-use Doctrine\ORM\Mapping as ORM;
+use Symfony\Component\Validator\Constraints as Asserts;
 
 /**
  * Dataflow execution status.
  *
- * @ORM\Entity(repositoryClass="CodeRhapsodie\DataflowBundle\Repository\JobRepository")
- * @ORM\Table(name="cr_dataflow_job")
+ * @codeCoverageIgnore
  */
 class Job
 {
-    const STATUS_PENDING = 0;
-    const STATUS_RUNNING = 1;
-    const STATUS_COMPLETED = 2;
+    public const STATUS_PENDING = 0;
+    public const STATUS_RUNNING = 1;
+    public const STATUS_COMPLETED = 2;
+    public const STATUS_QUEUED = 3;
 
-    /**
-     * @var int
-     *
-     * @ORM\Id()
-     * @ORM\Column(type="integer")
-     * @ORM\GeneratedValue(strategy="AUTO")
-     */
-    private $id;
+    private const KEYS = [
+        'id',
+        'status',
+        'label',
+        'dataflow_type',
+        'options',
+        'requested_date',
+        'scheduled_dataflow_id',
+        'count',
+        'exceptions',
+        'start_time',
+        'end_time',
+    ];
 
-    /**
-     * @var int
-     *
-     * @ORM\Column(type="integer")
-     */
-    private $status;
+    private ?int $id = null;
 
-    /**
-     * @var string|null
-     *
-     * @ORM\Column(type="string")
-     */
-    private $label;
+    #[Asserts\Range(min: 0, max: 2)]
+    private int $status = self::STATUS_PENDING;
 
-    /**
-     * @var string|null
-     *
-     * @ORM\Column(type="string")
-     */
-    private $dataflowType;
+    #[Asserts\NotBlank]
+    #[Asserts\Length(min: 1, max: 255)]
+    #[Asserts\Regex('#^[[:alnum:] ]+\z#u')]
+    private ?string $label = null;
 
-    /**
-     * @var array|null
-     *
-     * @ORM\Column(type="json")
-     */
-    private $options;
+    #[Asserts\NotBlank]
+    #[Asserts\Length(min: 1, max: 255)]
+    #[Asserts\Regex('#^[[:alnum:]\\\]+\z#u')]
+    private ?string $dataflowType = null;
 
-    /**
-     * @var \DateTimeInterface|null
-     *
-     * @ORM\Column(type="datetime", nullable=true)
-     */
-    private $requestedDate;
+    private ?array $options = null;
 
-    /**
-     * @var ScheduledDataflow|null
-     *
-     * @ORM\ManyToOne(targetEntity="ScheduledDataflow", inversedBy="jobs")
-     * @ORM\JoinColumn(nullable=true)
-     */
-    private $scheduledDataflow;
+    private ?\DateTimeInterface $requestedDate = null;
 
-    /**
-     * @var int|null
-     *
-     * @ORM\Column(type="integer", nullable=true)
-     */
-    private $count;
+    private ?int $scheduledDataflowId = null;
 
-    /**
-     * @var array|null
-     *
-     * @ORM\Column(type="json", nullable=true)
-     */
-    private $exceptions;
+    private ?int $count = 0;
 
-    /**
-     * @var \DateTimeInterface|null
-     *
-     * @ORM\Column(type="datetime", nullable=true)
-     */
-    private $startTime;
+    private ?array $exceptions = null;
 
-    /**
-     * @var \DateTimeInterface|null
-     *
-     * @ORM\Column(type="datetime", nullable=true)
-     */
-    private $endTime;
+    private ?\DateTimeInterface $startTime = null;
+
+    private ?\DateTimeInterface $endTime = null;
 
-    /**
-     * @param ScheduledDataflow $scheduled
-     *
-     * @return Job
-     */
     public static function createFromScheduledDataflow(ScheduledDataflow $scheduled): self
     {
         return (new static())
@@ -111,31 +69,70 @@ class Job
             ->setOptions($scheduled->getOptions())
             ->setRequestedDate(clone $scheduled->getNext())
             ->setLabel($scheduled->getLabel())
-            ->setScheduledDataflow($scheduled)
-        ;
+            ->setScheduledDataflowId($scheduled->getId());
     }
 
-    /**
-     * @return int
-     */
-    public function getId(): int
+    public function __construct()
+    {
+    }
+
+    public static function createFromArray(array $datas)
+    {
+        $lost = array_diff(static::KEYS, array_keys($datas));
+        if (count($lost) > 0) {
+            throw new \LogicException('The first argument of '.__METHOD__.'  must be contains: "'.implode(', ', $lost).'"');
+        }
+
+        $job = new self();
+        $job->id = null === $datas['id'] ? null : (int) $datas['id'];
+        $job->setStatus(null === $datas['status'] ? null : (int) $datas['status']);
+        $job->setLabel($datas['label']);
+        $job->setDataflowType($datas['dataflow_type']);
+        $job->setOptions($datas['options']);
+        $job->setRequestedDate($datas['requested_date']);
+        $job->setScheduledDataflowId(null === $datas['scheduled_dataflow_id'] ? null : (int) $datas['scheduled_dataflow_id']);
+        $job->setCount(null === $datas['count'] ? null : (int) $datas['count']);
+        $job->setExceptions($datas['exceptions']);
+        $job->setStartTime($datas['start_time']);
+        $job->setEndTime($datas['end_time']);
+
+        return $job;
+    }
+
+    public function toArray(): array
+    {
+        return [
+            'id' => $this->getId(),
+            'status' => $this->getStatus(),
+            'label' => $this->getLabel(),
+            'dataflow_type' => $this->getDataflowType(),
+            'options' => $this->getOptions(),
+            'requested_date' => $this->getRequestedDate(),
+            'scheduled_dataflow_id' => $this->getScheduledDataflowId(),
+            'count' => $this->getCount(),
+            'exceptions' => $this->getExceptions(),
+            'start_time' => $this->getStartTime(),
+            'end_time' => $this->getEndTime(),
+        ];
+    }
+
+    public function setId(int $id): Job
+    {
+        $this->id = $id;
+
+        return $this;
+    }
+
+    public function getId(): ?int
     {
         return $this->id;
     }
 
-    /**
-     * @return int
-     */
     public function getStatus(): int
     {
         return $this->status;
     }
 
-    /**
-     * @param int $status
-     *
-     * @return Job
-     */
     public function setStatus(int $status): Job
     {
         $this->status = $status;
@@ -143,19 +140,11 @@ class Job
         return $this;
     }
 
-    /**
-     * @return null|string
-     */
     public function getLabel(): ?string
     {
         return $this->label;
     }
 
-    /**
-     * @param null|string $label
-     *
-     * @return Job
-     */
     public function setLabel(?string $label): Job
     {
         $this->label = $label;
@@ -163,19 +152,11 @@ class Job
         return $this;
     }
 
-    /**
-     * @return null|string
-     */
     public function getDataflowType(): ?string
     {
         return $this->dataflowType;
     }
 
-    /**
-     * @param null|string $dataflowType
-     *
-     * @return Job
-     */
     public function setDataflowType(?string $dataflowType): Job
     {
         $this->dataflowType = $dataflowType;
@@ -183,19 +164,11 @@ class Job
         return $this;
     }
 
-    /**
-     * @return array|null
-     */
     public function getOptions(): ?array
     {
         return $this->options;
     }
 
-    /**
-     * @param array|null $options
-     *
-     * @return Job
-     */
     public function setOptions(?array $options): Job
     {
         $this->options = $options;
@@ -203,19 +176,11 @@ class Job
         return $this;
     }
 
-    /**
-     * @return \DateTimeInterface|null
-     */
     public function getRequestedDate(): ?\DateTimeInterface
     {
         return $this->requestedDate;
     }
 
-    /**
-     * @param \DateTimeInterface|null $requestedDate
-     *
-     * @return Job
-     */
     public function setRequestedDate(?\DateTimeInterface $requestedDate): Job
     {
         $this->requestedDate = $requestedDate;
@@ -223,39 +188,23 @@ class Job
         return $this;
     }
 
-    /**
-     * @return ScheduledDataflow|null
-     */
-    public function getScheduledDataflow(): ?ScheduledDataflow
+    public function getScheduledDataflowId(): ?int
     {
-        return $this->scheduledDataflow;
+        return $this->scheduledDataflowId;
     }
 
-    /**
-     * @param ScheduledDataflow|null $scheduledDataflow
-     *
-     * @return Job
-     */
-    public function setScheduledDataflow(?ScheduledDataflow $scheduledDataflow): Job
+    public function setScheduledDataflowId(?int $scheduledDataflowId): Job
     {
-        $this->scheduledDataflow = $scheduledDataflow;
+        $this->scheduledDataflowId = $scheduledDataflowId;
 
         return $this;
     }
 
-    /**
-     * @return int|null
-     */
     public function getCount(): ?int
     {
         return $this->count;
     }
 
-    /**
-     * @param int|null $count
-     *
-     * @return Job
-     */
     public function setCount(?int $count): Job
     {
         $this->count = $count;
@@ -263,19 +212,11 @@ class Job
         return $this;
     }
 
-    /**
-     * @return array|null
-     */
     public function getExceptions(): ?array
     {
         return $this->exceptions;
     }
 
-    /**
-     * @param array|null $exceptions
-     *
-     * @return Job
-     */
     public function setExceptions(?array $exceptions): Job
     {
         $this->exceptions = $exceptions;
@@ -283,19 +224,11 @@ class Job
         return $this;
     }
 
-    /**
-     * @return \DateTimeInterface|null
-     */
     public function getStartTime(): ?\DateTimeInterface
     {
         return $this->startTime;
     }
 
-    /**
-     * @param \DateTimeInterface|null $startTime
-     *
-     * @return Job
-     */
     public function setStartTime(?\DateTimeInterface $startTime): Job
     {
         $this->startTime = $startTime;
@@ -303,19 +236,11 @@ class Job
         return $this;
     }
 
-    /**
-     * @return \DateTimeInterface|null
-     */
     public function getEndTime(): ?\DateTimeInterface
     {
         return $this->endTime;
     }
 
-    /**
-     * @param \DateTimeInterface|null $endTime
-     *
-     * @return Job
-     */
     public function setEndTime(?\DateTimeInterface $endTime): Job
     {
         $this->endTime = $endTime;

src/Entity/ScheduledDataflow.php

@@ -5,105 +5,98 @@ declare(strict_types=1);
 namespace CodeRhapsodie\DataflowBundle\Entity;
 
 use CodeRhapsodie\DataflowBundle\Validator\Constraints\Frequency;
-use Doctrine\ORM\Mapping as ORM;
+use Symfony\Component\Validator\Constraints as Asserts;
 
 /**
  * Schedule for a regular execution of a dataflow.
  *
- * @ORM\Entity(repositoryClass="CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository")
- * @ORM\Table(name="cr_dataflow_scheduled")
+ * @codeCoverageIgnore
  */
 class ScheduledDataflow
 {
-    const AVAILABLE_FREQUENCIES = [
+    public const AVAILABLE_FREQUENCIES = [
         '1 hour',
         '1 day',
         '1 week',
         '1 month',
     ];
 
-    /**
-     * @var int
-     *
-     * @ORM\Id()
-     * @ORM\Column(name="id", type="integer")
-     * @ORM\GeneratedValue(strategy="AUTO")
-     */
-    private $id;
+    private const KEYS = ['id', 'label', 'dataflow_type', 'options', 'frequency', 'next', 'enabled'];
+
+    private ?int $id = null;
+
+    #[Asserts\NotBlank]
+    #[Asserts\Length(min: 1, max: 255)]
+    #[Asserts\Regex('#^[[:alnum:] ]+\z#u')]
+    private ?string $label = null;
+
+    #[Asserts\NotBlank]
+    #[Asserts\Length(min: 1, max: 255)]
+    #[Asserts\Regex('#^[[:alnum:]\\\]+\z#u')]
+    private ?string $dataflowType = null;
+
+    private ?array $options = null;
 
     /**
-     * @var string|null
-     *
-     * @ORM\Column(type="string")
-     */
-    private $label;
-
-    /**
-     * @var string|null
-     *
-     * @ORM\Column(type="string")
-     */
-    private $dataflowType;
-
-    /**
-     * @var array|null
-     *
-     * @ORM\Column(type="json")
-     */
-    private $options;
-
-    /**
-     * @var string|null
-     *
-     * @ORM\Column(type="string")
-     *
      * @Frequency()
      */
-    private $frequency;
+    #[Asserts\NotBlank]
+    private ?string $frequency = null;
 
-    /**
-     * @var \DateTimeInterface|null
-     *
-     * @ORM\Column(type="datetime", nullable=true)
-     */
-    private $next;
+    private ?\DateTimeInterface $next = null;
 
-    /**
-     * @var bool|null
-     *
-     * @ORM\Column(type="boolean")
-     */
-    private $enabled;
+    private ?bool $enabled = null;
 
-    /**
-     * @var Job[]
-     *
-     * @ORM\OneToMany(targetEntity="Job", mappedBy="scheduledDataflow", cascade={"persist"})
-     * @ORM\OrderBy({"startTime" = "DESC"})
-     */
-    private $jobs;
+    public static function createFromArray(array $datas)
+    {
+        $lost = array_diff(static::KEYS, array_keys($datas));
+        if (count($lost) > 0) {
+            throw new \LogicException('The first argument of '.__METHOD__.'  must be contains: "'.implode(', ', $lost).'"');
+        }
 
-    /**
-     * @return int
-     */
-    public function getId(): int
+        $scheduledDataflow = new self();
+        $scheduledDataflow->id = null === $datas['id'] ? null : (int) $datas['id'];
+
+        $scheduledDataflow->setLabel($datas['label']);
+        $scheduledDataflow->setDataflowType($datas['dataflow_type']);
+        $scheduledDataflow->setOptions($datas['options']);
+        $scheduledDataflow->setFrequency($datas['frequency']);
+        $scheduledDataflow->setNext($datas['next']);
+        $scheduledDataflow->setEnabled(null === $datas['enabled'] ? null : (bool) $datas['enabled']);
+
+        return $scheduledDataflow;
+    }
+
+    public function toArray(): array
+    {
+        return [
+            'id' => $this->getId(),
+            'label' => $this->getLabel(),
+            'dataflow_type' => $this->getDataflowType(),
+            'options' => $this->getOptions(),
+            'frequency' => $this->getFrequency(),
+            'next' => $this->getNext(),
+            'enabled' => $this->getEnabled(),
+        ];
+    }
+
+    public function setId(int $id): ScheduledDataflow
+    {
+        $this->id = $id;
+
+        return $this;
+    }
+
+    public function getId(): ?int
     {
         return $this->id;
     }
 
-    /**
-     * @return null|string
-     */
     public function getLabel(): ?string
     {
         return $this->label;
     }
 
-    /**
-     * @param null|string $label
-     *
-     * @return ScheduledDataflow
-     */
     public function setLabel(?string $label): ScheduledDataflow
     {
         $this->label = $label;
@@ -111,19 +104,11 @@ class ScheduledDataflow
         return $this;
     }
 
-    /**
-     * @return null|string
-     */
     public function getDataflowType(): ?string
     {
         return $this->dataflowType;
     }
 
-    /**
-     * @param null|string $dataflowType
-     *
-     * @return ScheduledDataflow
-     */
     public function setDataflowType(?string $dataflowType): ScheduledDataflow
     {
         $this->dataflowType = $dataflowType;
@@ -131,19 +116,11 @@ class ScheduledDataflow
         return $this;
     }
 
-    /**
-     * @return array|null
-     */
     public function getOptions(): ?array
     {
         return $this->options;
     }
 
-    /**
-     * @param array|null $options
-     *
-     * @return ScheduledDataflow
-     */
     public function setOptions(?array $options): ScheduledDataflow
     {
         $this->options = $options;
@@ -151,19 +128,11 @@ class ScheduledDataflow
         return $this;
     }
 
-    /**
-     * @return string|null
-     */
     public function getFrequency(): ?string
     {
         return $this->frequency;
     }
 
-    /**
-     * @param string|null $frequency
-     *
-     * @return ScheduledDataflow
-     */
     public function setFrequency(?string $frequency): ScheduledDataflow
     {
         $this->frequency = $frequency;
@@ -171,19 +140,11 @@ class ScheduledDataflow
         return $this;
     }
 
-    /**
-     * @return \DateTimeInterface|null
-     */
     public function getNext(): ?\DateTimeInterface
     {
         return $this->next;
     }
 
-    /**
-     * @param \DateTimeInterface|null $next
-     *
-     * @return ScheduledDataflow
-     */
     public function setNext(?\DateTimeInterface $next): ScheduledDataflow
     {
         $this->next = $next;
@@ -191,19 +152,11 @@ class ScheduledDataflow
         return $this;
     }
 
-    /**
-     * @return bool|null
-     */
     public function getEnabled(): ?bool
     {
         return $this->enabled;
     }
 
-    /**
-     * @param bool|null $enabled
-     *
-     * @return ScheduledDataflow
-     */
     public function setEnabled(?bool $enabled): ScheduledDataflow
     {
         $this->enabled = $enabled;

src/Event/CrEvent.php

@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Event;
+
+/*
+ * @codeCoverageIgnore
+ */
+if (class_exists(\Symfony\Contracts\EventDispatcher\Event::class)) {
+    // For Symfony 5.0+
+    abstract class CrEvent extends \Symfony\Contracts\EventDispatcher\Event
+    {
+    }
+} else {
+    // For Symfony 3.4 to 4.4
+    abstract class CrEvent extends \Symfony\Component\EventDispatcher\Event
+    {
+    }
+}

src/Event/Events.php

@@ -6,6 +6,6 @@ namespace CodeRhapsodie\DataflowBundle\Event;
 
 final class Events
 {
-    const BEFORE_PROCESSING = 'coderhapsodie.dataflow.before_processing';
-    const AFTER_PROCESSING = 'coderhapsodie.dataflow.after_processing';
+    public const BEFORE_PROCESSING = 'coderhapsodie.dataflow.before_processing';
+    public const AFTER_PROCESSING = 'coderhapsodie.dataflow.after_processing';
 }

src/Event/ProcessingEvent.php

@@ -5,29 +5,21 @@ declare(strict_types=1);
 namespace CodeRhapsodie\DataflowBundle\Event;
 
 use CodeRhapsodie\DataflowBundle\Entity\Job;
-use Symfony\Component\EventDispatcher\Event;
 
 /**
  * Event used during the dataflow lifecycle.
+ *
+ * @codeCoverageIgnore
  */
-class ProcessingEvent extends Event
+class ProcessingEvent extends CrEvent
 {
-    /** @var Job */
-    private $job;
-
     /**
      * ProcessingEvent constructor.
-     *
-     * @param Job $job
      */
-    public function __construct(Job $job)
+    public function __construct(private Job $job)
     {
-        $this->job = $job;
     }
 
-    /**
-     * @return Job
-     */
     public function getJob(): Job
     {
         return $this->job;

src/Exceptions/UnknownDataflowTypeException.php

@@ -9,4 +9,12 @@ namespace CodeRhapsodie\DataflowBundle\Exceptions;
  */
 class UnknownDataflowTypeException extends \Exception
 {
+    public static function create(string $aliasOrFqcn, array $knownDataflowTypes): self
+    {
+        return new self(sprintf(
+            'Unknown dataflow type FQCN or alias "%s". Registered dataflow types FQCN and aliases are %s.',
+            $aliasOrFqcn,
+            implode(', ', $knownDataflowTypes)
+        ));
+    }
 }

src/Exceptions/UnsupportedItemTypeException.php

@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Exceptions;
+
+/**
+ * Exception thrown when a writer receives an item of an unsupported type.
+ */
+class UnsupportedItemTypeException extends \Exception
+{
+}

src/Factory/ConnectionFactory.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Factory;
+
+use Symfony\Component\DependencyInjection\Container;
+
+/**
+ * Class ConnectionFactory.
+ *
+ * @codeCoverageIgnore
+ */
+class ConnectionFactory
+{
+    public function __construct(private Container $container, private string $connectionName)
+    {
+    }
+
+    public function setConnectionName(string $connectionName)
+    {
+        $this->connectionName = $connectionName;
+    }
+
+    public function getConnection(): \Doctrine\DBAL\Connection
+    {
+        return $this->container->get(sprintf('doctrine.dbal.%s_connection', $this->connectionName));
+    }
+}

src/Logger/BufferHandler.php

@@ -0,0 +1,40 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Logger;
+
+use Monolog\Formatter\FormatterInterface;
+use Monolog\Formatter\LineFormatter;
+use Monolog\Handler\AbstractProcessingHandler;
+use Monolog\Logger;
+
+class BufferHandler extends AbstractProcessingHandler
+{
+    private const FORMAT = "[%datetime%] %level_name% when processing item %context.index%: %message% %context% %extra%\n";
+
+    private array $buffer = [];
+
+    public function __construct($level = Logger::DEBUG, bool $bubble = true)
+    {
+        parent::__construct($level, $bubble);
+    }
+
+    public function clearBuffer(): array
+    {
+        $logs = $this->buffer;
+        $this->buffer = [];
+
+        return $logs;
+    }
+
+    protected function write(array $record): void
+    {
+        $this->buffer[] = $record['formatted'];
+    }
+
+    protected function getDefaultFormatter(): FormatterInterface
+    {
+        return new LineFormatter(self::FORMAT);
+    }
+}

src/Logger/DelegatingLogger.php

@@ -0,0 +1,32 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Logger;
+
+use Psr\Log\AbstractLogger;
+use Psr\Log\LoggerInterface;
+
+final class DelegatingLogger extends AbstractLogger
+{
+    /** @var LoggerInterface[] */
+    private ?array $loggers = null;
+
+    public function __construct(iterable $loggers)
+    {
+        foreach ($loggers as $logger) {
+            if (!$logger instanceof LoggerInterface) {
+                throw new \InvalidArgumentException(sprintf('Only instances of %s should be passed to the constructor of %s. An instance of %s was passed instead.', LoggerInterface::class, self::class, $logger::class));
+            }
+
+            $this->loggers[] = $logger;
+        }
+    }
+
+    public function log($level, $message, array $context = []): void
+    {
+        foreach ($this->loggers as $logger) {
+            $logger->log($level, $message, $context);
+        }
+    }
+}

src/Manager/ScheduledDataflowManager.php

@@ -4,31 +4,19 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\Manager;
 
-use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
 use CodeRhapsodie\DataflowBundle\Entity\Job;
-use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
+use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
 use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
-use Doctrine\ORM\EntityManagerInterface;
+use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
+use Doctrine\DBAL\Connection;
 
 /**
  * Handles scheduled dataflows execution dates based on their frequency.
  */
 class ScheduledDataflowManager implements ScheduledDataflowManagerInterface
 {
-    /** @var EntityManagerInterface */
-    private $em;
-
-    /** @var ScheduledDataflowRepository */
-    private $scheduledDataflowRepository;
-
-    /** @var JobRepository */
-    private $jobRepository;
-
-    public function __construct(EntityManagerInterface $em, ScheduledDataflowRepository $scheduledDataflowRepository, JobRepository $jobRepository)
+    public function __construct(private Connection $connection, private ScheduledDataflowRepository $scheduledDataflowRepository, private JobRepository $jobRepository)
     {
-        $this->em = $em;
-        $this->scheduledDataflowRepository = $scheduledDataflowRepository;
-        $this->jobRepository = $jobRepository;
     }
 
     /**
@@ -36,21 +24,23 @@ class ScheduledDataflowManager implements ScheduledDataflowManagerInterface
      */
     public function createJobsFromScheduledDataflows(): void
     {
-        foreach ($this->scheduledDataflowRepository->findReadyToRun() as $scheduled) {
-            if (null !== $this->jobRepository->findPendingForScheduledDataflow($scheduled)) {
-                continue;
+        $this->connection->beginTransaction();
+        try {
+            foreach ($this->scheduledDataflowRepository->findReadyToRun() as $scheduled) {
+                if (null !== $this->jobRepository->findPendingForScheduledDataflow($scheduled)) {
+                    continue;
+                }
+
+                $this->createPendingForScheduled($scheduled);
+                $this->updateScheduledDataflowNext($scheduled);
             }
-
-            $this->createPendingForScheduled($scheduled);
-            $this->updateScheduledDataflowNext($scheduled);
+        } catch (\Throwable $e) {
+            $this->connection->rollBack();
+            throw $e;
         }
-
-        $this->em->flush();
+        $this->connection->commit();
     }
 
-    /**
-     * @param ScheduledDataflow $scheduled
-     */
     private function updateScheduledDataflowNext(ScheduledDataflow $scheduled): void
     {
         $interval = \DateInterval::createFromDateString($scheduled->getFrequency());
@@ -62,13 +52,11 @@ class ScheduledDataflowManager implements ScheduledDataflowManagerInterface
         }
 
         $scheduled->setNext($next);
+        $this->scheduledDataflowRepository->save($scheduled);
     }
 
-    /**
-     * @param ScheduledDataflow $scheduled
-     */
     private function createPendingForScheduled(ScheduledDataflow $scheduled): void
     {
-        $this->em->persist(Job::createFromScheduledDataflow($scheduled));
+        $this->jobRepository->save(Job::createFromScheduledDataflow($scheduled));
     }
 }

src/MessengerMode/JobMessage.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\MessengerMode;
+
+class JobMessage
+{
+    public function __construct(private int $jobId)
+    {
+    }
+
+    public function getJobId(): int
+    {
+        return $this->jobId;
+    }
+}

src/MessengerMode/JobMessageHandler.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\MessengerMode;
+
+use CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface;
+use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
+use Symfony\Component\Messenger\Handler\MessageSubscriberInterface;
+
+class JobMessageHandler implements MessageSubscriberInterface
+{
+    public function __construct(private JobRepository $repository, private JobProcessorInterface $processor)
+    {
+    }
+
+    public function __invoke(JobMessage $message)
+    {
+        $this->processor->process($this->repository->find($message->getJobId()));
+    }
+
+    public static function getHandledMessages(): iterable
+    {
+        return [JobMessage::class];
+    }
+}

src/Processor/JobProcessor.php

@@ -0,0 +1,87 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Processor;
+
+use CodeRhapsodie\DataflowBundle\DataflowType\Result;
+use CodeRhapsodie\DataflowBundle\Entity\Job;
+use CodeRhapsodie\DataflowBundle\Event\Events;
+use CodeRhapsodie\DataflowBundle\Event\ProcessingEvent;
+use CodeRhapsodie\DataflowBundle\Logger\BufferHandler;
+use CodeRhapsodie\DataflowBundle\Logger\DelegatingLogger;
+use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface;
+use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
+use Monolog\Logger;
+use Psr\Log\LoggerAwareInterface;
+use Psr\Log\LoggerAwareTrait;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+
+class JobProcessor implements JobProcessorInterface, LoggerAwareInterface
+{
+    use LoggerAwareTrait;
+
+    public function __construct(private JobRepository $repository, private DataflowTypeRegistryInterface $registry, private EventDispatcherInterface $dispatcher)
+    {
+    }
+
+    public function process(Job $job): void
+    {
+        $this->beforeProcessing($job);
+
+        $dataflowType = $this->registry->getDataflowType($job->getDataflowType());
+        $loggers = [new Logger('dataflow_internal', [$bufferHandler = new BufferHandler()])];
+        if (isset($this->logger)) {
+            $loggers[] = $this->logger;
+        }
+        $logger = new DelegatingLogger($loggers);
+
+        if ($dataflowType instanceof LoggerAwareInterface) {
+            $dataflowType->setLogger($logger);
+        }
+
+        $result = $dataflowType->process($job->getOptions());
+
+        if (!$dataflowType instanceof LoggerAwareInterface) {
+            foreach ($result->getExceptions() as $index => $e) {
+                $logger->error($e, ['index' => $index]);
+            }
+        }
+
+        $this->afterProcessing($job, $result, $bufferHandler);
+    }
+
+    private function beforeProcessing(Job $job): void
+    {
+        // Symfony 3.4 to 4.4 call
+        if (!class_exists(\Symfony\Contracts\EventDispatcher\Event::class)) {
+            $this->dispatcher->dispatch(Events::BEFORE_PROCESSING, new ProcessingEvent($job));
+        } else { // Symfony 5.0+ call
+            $this->dispatcher->dispatch(new ProcessingEvent($job), Events::BEFORE_PROCESSING);
+        }
+
+        $job
+            ->setStatus(Job::STATUS_RUNNING)
+            ->setStartTime(new \DateTime())
+        ;
+        $this->repository->save($job);
+    }
+
+    private function afterProcessing(Job $job, Result $result, BufferHandler $bufferLogger): void
+    {
+        $job
+            ->setEndTime($result->getEndTime())
+            ->setStatus(Job::STATUS_COMPLETED)
+            ->setCount($result->getSuccessCount())
+            ->setExceptions($bufferLogger->clearBuffer())
+        ;
+        $this->repository->save($job);
+
+        // Symfony 3.4 to 4.4 call
+        if (!class_exists(\Symfony\Contracts\EventDispatcher\Event::class)) {
+            $this->dispatcher->dispatch(Events::AFTER_PROCESSING, new ProcessingEvent($job));
+        } else { // Symfony 5.0+ call
+            $this->dispatcher->dispatch(new ProcessingEvent($job), Events::AFTER_PROCESSING);
+        }
+    }
+}

src/Processor/JobProcessorInterface.php

@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Processor;
+
+use CodeRhapsodie\DataflowBundle\Entity\Job;
+
+interface JobProcessorInterface
+{
+    public function process(Job $job): void;
+}

src/Registry/DataflowTypeRegistry.php

@@ -4,8 +4,8 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\Registry;
 
-use CodeRhapsodie\DataflowBundle\Exceptions\UnknownDataflowTypeException;
 use CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface;
+use CodeRhapsodie\DataflowBundle\Exceptions\UnknownDataflowTypeException;
 
 /**
  * Array based dataflow types registry.
@@ -13,10 +13,10 @@ use CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface;
 class DataflowTypeRegistry implements DataflowTypeRegistryInterface
 {
     /** @var array|DataflowTypeInterface[] */
-    private $fqcnRegistry = [];
+    private array $fqcnRegistry = [];
 
     /** @var array|DataflowTypeInterface[] */
-    private $aliasesRegistry = [];
+    private array $aliasesRegistry = [];
 
     /**
      * {@inheritdoc}
@@ -31,7 +31,7 @@ class DataflowTypeRegistry implements DataflowTypeRegistryInterface
             return $this->aliasesRegistry[$fqcnOrAlias];
         }
 
-        throw new UnknownDataflowTypeException();
+        throw UnknownDataflowTypeException::create($fqcnOrAlias, [...array_keys($this->fqcnRegistry), ...array_keys($this->aliasesRegistry)]);
     }
 
     /**
@@ -47,7 +47,7 @@ class DataflowTypeRegistry implements DataflowTypeRegistryInterface
      */
     public function registerDataflowType(DataflowTypeInterface $dataflowType): void
     {
-        $this->fqcnRegistry[get_class($dataflowType)] = $dataflowType;
+        $this->fqcnRegistry[$dataflowType::class] = $dataflowType;
         foreach ($dataflowType->getAliases() as $alias) {
             $this->aliasesRegistry[$alias] = $dataflowType;
         }

src/Registry/DataflowTypeRegistryInterface.php

@@ -13,10 +13,6 @@ interface DataflowTypeRegistryInterface
 {
     /**
      * Get a registered dataflow type from its FQCN or one of its aliases.
-     *
-     * @param string $fqcnOrAlias
-     *
-     * @return DataflowTypeInterface
      */
     public function getDataflowType(string $fqcnOrAlias): DataflowTypeInterface;
 
@@ -29,8 +25,6 @@ interface DataflowTypeRegistryInterface
 
     /**
      * Registers a dataflow type.
-     *
-     * @param DataflowTypeInterface $dataflowType
      */
     public function registerDataflowType(DataflowTypeInterface $dataflowType): void;
 }

src/Repository/InitFromDbTrait.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Repository;
+
+/**
+ * @codeCoverageIgnore
+ */
+trait InitFromDbTrait
+{
+    private function initDateTime(array $datas): array
+    {
+        foreach (static::FIELDS_TYPE as $key => $type) {
+            if ('datetime' === $type && null !== $datas[$key]) {
+                $datas[$key] = new \DateTime($datas[$key]);
+            }
+        }
+
+        return $datas;
+    }
+
+    private function initArray(array $datas): array
+    {
+        if (!is_array($datas['options'])) {
+            $datas['options'] = $this->strToArray($datas['options']);
+        }
+        if (array_key_exists('exceptions', $datas) && !is_array($datas['exceptions'])) {
+            $datas['exceptions'] = $this->strToArray($datas['exceptions']);
+        }
+
+        return $datas;
+    }
+
+    private function strToArray($value): array
+    {
+        if (null === $value) {
+            return [];
+        }
+
+        $array = json_decode($value, true, 512, JSON_THROW_ON_ERROR);
+
+        return (false === $array) ? [] : $array;
+    }
+}

src/Repository/JobRepository.php

@@ -4,62 +4,169 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\Repository;
 
-use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
 use CodeRhapsodie\DataflowBundle\Entity\Job;
-use Doctrine\Common\Collections\Criteria;
-use Doctrine\ORM\EntityRepository;
+use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
+use Doctrine\DBAL\Connection;
+use Doctrine\DBAL\ParameterType;
+use Doctrine\DBAL\Query\QueryBuilder;
 
 /**
  * Repository.
+ *
+ * @codeCoverageIgnore
  */
-class JobRepository extends EntityRepository
+class JobRepository
 {
+    use InitFromDbTrait;
+
+    public const TABLE_NAME = 'cr_dataflow_job';
+
+    private const FIELDS_TYPE = [
+        'id' => ParameterType::INTEGER,
+        'status' => ParameterType::INTEGER,
+        'label' => ParameterType::STRING,
+        'dataflow_type' => ParameterType::STRING,
+        'options' => ParameterType::STRING,
+        'requested_date' => 'datetime',
+        'scheduled_dataflow_id' => ParameterType::INTEGER,
+        'count' => ParameterType::INTEGER,
+        'exceptions' => ParameterType::STRING,
+        'start_time' => 'datetime',
+        'end_time' => 'datetime',
+    ];
+
+    /**
+     * JobRepository constructor.
+     */
+    public function __construct(private Connection $connection)
+    {
+    }
+
+    public function find(int $jobId)
+    {
+        $qb = $this->createQueryBuilder();
+        $qb
+            ->andWhere($qb->expr()->eq('id', $qb->createNamedParameter($jobId, ParameterType::INTEGER)))
+        ;
+
+        return $this->returnFirstOrNull($qb);
+    }
+
     public function findOneshotDataflows(): iterable
     {
-        return $this->findBy([
-            'scheduledDataflow' => null,
-            'status' => Job::STATUS_PENDING,
-        ]);
+        $qb = $this->createQueryBuilder();
+        $qb
+            ->andWhere($qb->expr()->isNull('scheduled_dataflow_id'))
+            ->andWhere($qb->expr()->eq('status', $qb->createNamedParameter(Job::STATUS_PENDING, ParameterType::INTEGER)));
+        $stmt = $qb->execute();
+        if (0 === $stmt->rowCount()) {
+            return [];
+        }
+        while (false !== ($row = $stmt->fetchAssociative())) {
+            yield Job::createFromArray($this->initDateTime($this->initArray($row)));
+        }
     }
 
     public function findPendingForScheduledDataflow(ScheduledDataflow $scheduled): ?Job
     {
-        return $this->findOneBy([
-            'scheduledDataflow' => $scheduled->getId(),
-            'status' => Job::STATUS_PENDING,
-        ]);
+        $qb = $this->createQueryBuilder();
+        $qb
+            ->andWhere($qb->expr()->eq('scheduled_dataflow_id', $qb->createNamedParameter($scheduled->getId(), ParameterType::INTEGER)))
+            ->andWhere($qb->expr()->eq('status', $qb->createNamedParameter(Job::STATUS_PENDING, ParameterType::INTEGER)));
+
+        return $this->returnFirstOrNull($qb);
     }
 
     public function findNextPendingDataflow(): ?Job
     {
-        $criteria = (new Criteria())
-            ->where(Criteria::expr()->lte('requestedDate', new \DateTime()))
-            ->andWhere(Criteria::expr()->eq('status', Job::STATUS_PENDING))
-            ->orderBy(['requestedDate' => Criteria::ASC])
+        $qb = $this->createQueryBuilder();
+        $qb->andWhere($qb->expr()->lte('requested_date', $qb->createNamedParameter(new \DateTime(), 'datetime')))
+            ->andWhere($qb->expr()->eq('status', $qb->createNamedParameter(Job::STATUS_PENDING, ParameterType::INTEGER)))
+            ->orderBy('requested_date', 'ASC')
             ->setMaxResults(1)
         ;
 
-        return $this->matching($criteria)->first() ?: null;
+        return $this->returnFirstOrNull($qb);
     }
 
     public function findLastForDataflowId(int $dataflowId): ?Job
     {
-        return $this->findOneBy(['scheduledDataflow' => $dataflowId], ['requestedDate' => 'desc']);
+        $qb = $this->createQueryBuilder();
+        $qb->andWhere($qb->expr()->eq('scheduled_dataflow_id', $qb->createNamedParameter($dataflowId, ParameterType::INTEGER)))
+            ->orderBy('requested_date', 'DESC')
+            ->setMaxResults(1)
+        ;
+
+        return $this->returnFirstOrNull($qb);
     }
 
     public function findLatests(): iterable
     {
-        return $this->findBy([], ['requestedDate' => 'desc'], 20);
+        $qb = $this->createQueryBuilder();
+        $qb
+            ->orderBy('requested_date', 'DESC')
+            ->setMaxResults(20);
+        $stmt = $qb->execute();
+        if (0 === $stmt->rowCount()) {
+            return [];
+        }
+        while (false !== ($row = $stmt->fetchAssociative())) {
+            yield Job::createFromArray($row);
+        }
     }
 
     public function findForScheduled(int $id): iterable
     {
-        return $this->findBy(['scheduledDataflow' => $id], ['requestedDate' => 'desc'], 20);
+        $qb = $this->createQueryBuilder();
+        $qb->andWhere($qb->expr()->eq('scheduled_dataflow_id', $qb->createNamedParameter($id, ParameterType::INTEGER)))
+            ->orderBy('requested_date', 'DESC')
+            ->setMaxResults(20);
+        $stmt = $qb->execute();
+        if (0 === $stmt->rowCount()) {
+            return [];
+        }
+        while (false !== ($row = $stmt->fetchAssociative())) {
+            yield Job::createFromArray($row);
+        }
     }
 
     public function save(Job $job)
     {
-        $this->_em->persist($job);
-        $this->_em->flush();
+        $datas = $job->toArray();
+        unset($datas['id']);
+
+        if (is_array($datas['options'])) {
+            $datas['options'] = json_encode($datas['options'], JSON_THROW_ON_ERROR);
+        }
+        if (is_array($datas['exceptions'])) {
+            $datas['exceptions'] = json_encode($datas['exceptions'], JSON_THROW_ON_ERROR);
+        }
+
+        if (null === $job->getId()) {
+            $this->connection->insert(static::TABLE_NAME, $datas, static::FIELDS_TYPE);
+            $job->setId((int) $this->connection->lastInsertId());
+
+            return;
+        }
+        $this->connection->update(static::TABLE_NAME, $datas, ['id' => $job->getId()], static::FIELDS_TYPE);
+    }
+
+    public function createQueryBuilder($alias = null): QueryBuilder
+    {
+        $qb = $this->connection->createQueryBuilder();
+        $qb->select('*')
+            ->from(static::TABLE_NAME, $alias);
+
+        return $qb;
+    }
+
+    private function returnFirstOrNull(QueryBuilder $qb): ?Job
+    {
+        $stmt = $qb->execute();
+        if (0 === $stmt->rowCount()) {
+            return null;
+        }
+
+        return Job::createFromArray($this->initDateTime($this->initArray($stmt->fetchAssociative())));
     }
 }

src/Repository/ScheduledDataflowRepository.php

@@ -5,14 +5,38 @@ declare(strict_types=1);
 namespace CodeRhapsodie\DataflowBundle\Repository;
 
 use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
-use Doctrine\Common\Collections\Criteria;
-use Doctrine\ORM\EntityRepository;
+use Doctrine\DBAL\Connection;
+use Doctrine\DBAL\ParameterType;
+use Doctrine\DBAL\Query\QueryBuilder;
 
 /**
  * Repository for the ScheduledDataflow entity.
+ *
+ * @codeCoverageIgnore
  */
-class ScheduledDataflowRepository extends EntityRepository
+class ScheduledDataflowRepository
 {
+    use InitFromDbTrait;
+
+    public const TABLE_NAME = 'cr_dataflow_scheduled';
+
+    private const FIELDS_TYPE = [
+        'id' => ParameterType::INTEGER,
+        'label' => ParameterType::STRING,
+        'dataflow_type' => ParameterType::STRING,
+        'options' => ParameterType::STRING,
+        'frequency' => ParameterType::STRING,
+        'next' => 'datetime',
+        'enabled' => ParameterType::BOOLEAN,
+    ];
+
+    /**
+     * JobRepository constructor.
+     */
+    public function __construct(private Connection $connection)
+    {
+    }
+
     /**
      * Finds all enabled scheduled dataflows with a passed next run date.
      *
@@ -20,42 +44,105 @@ class ScheduledDataflowRepository extends EntityRepository
      */
     public function findReadyToRun(): iterable
     {
-        $criteria = (new Criteria())
-            ->where(Criteria::expr()->lte('next', new \DateTime()))
-            ->andWhere(Criteria::expr()->eq('enabled', 1))
-            ->orderBy(['next' => Criteria::ASC])
+        $qb = $this->createQueryBuilder();
+        $qb->andWhere($qb->expr()->lte('next', $qb->createNamedParameter(new \DateTime(), 'datetime')))
+            ->andWhere($qb->expr()->eq('enabled', 1))
+            ->orderBy('next', 'ASC')
         ;
 
-        return $this->matching($criteria);
+        $stmt = $qb->execute();
+        if (0 === $stmt->rowCount()) {
+            return [];
+        }
+        while (false !== ($row = $stmt->fetchAssociative())) {
+            yield ScheduledDataflow::createFromArray($this->initDateTime($this->initArray($row)));
+        }
+    }
+
+    public function find(int $scheduleId): ?ScheduledDataflow
+    {
+        $qb = $this->createQueryBuilder();
+        $qb->andWhere($qb->expr()->eq('id', $qb->createNamedParameter($scheduleId, ParameterType::INTEGER)))
+            ->setMaxResults(1)
+        ;
+
+        return $this->returnFirstOrNull($qb);
     }
 
     public function findAllOrderedByLabel(): iterable
     {
-        return $this->findBy([], ['label' => 'asc']);
+        $qb = $this->createQueryBuilder();
+        $qb->orderBy('label', 'ASC');
+
+        $stmt = $qb->execute();
+        if (0 === $stmt->rowCount()) {
+            return [];
+        }
+        while (false !== ($row = $stmt->fetchAssociative())) {
+            yield ScheduledDataflow::createFromArray($this->initDateTime($this->initArray($row)));
+        }
     }
 
     public function listAllOrderedByLabel(): array
     {
-        $query = $this->createQueryBuilder('w')
-            ->select('w.id', 'w.label', 'w.enabled', 'w.next', 'max(j.startTime) as startTime')
-            ->leftJoin('w.jobs', 'j')
+        $query = $this->connection->createQueryBuilder()
+            ->from(static::TABLE_NAME, 'w')
+            ->select('w.id', 'w.label', 'w.enabled', 'w.next', 'max(j.start_time) as startTime')
+            ->leftJoin('w', JobRepository::TABLE_NAME, 'j', 'j.scheduled_dataflow_id = w.id')
             ->orderBy('w.label', 'ASC')
             ->groupBy('w.id');
 
-        return $query->getQuery()->execute();
+        return $query->execute()->fetchAllAssociative();
     }
 
     public function save(ScheduledDataflow $scheduledDataflow)
     {
-        $this->_em->persist($scheduledDataflow);
-        $this->_em->flush();
+        $datas = $scheduledDataflow->toArray();
+        unset($datas['id']);
+
+        if (is_array($datas['options'])) {
+            $datas['options'] = json_encode($datas['options'], JSON_THROW_ON_ERROR);
+        }
+
+        if (null === $scheduledDataflow->getId()) {
+            $this->connection->insert(static::TABLE_NAME, $datas, static::FIELDS_TYPE);
+            $scheduledDataflow->setId((int) $this->connection->lastInsertId());
+
+            return;
+        }
+        $this->connection->update(static::TABLE_NAME, $datas, ['id' => $scheduledDataflow->getId()], static::FIELDS_TYPE);
     }
 
     public function delete(int $id): void
     {
-        $dataflow = $this->find($id);
+        $this->connection->beginTransaction();
+        try {
+            $this->connection->delete(JobRepository::TABLE_NAME, ['scheduled_dataflow_id' => $id]);
+            $this->connection->delete(static::TABLE_NAME, ['id' => $id]);
+        } catch (\Throwable $e) {
+            $this->connection->rollBack();
+            throw $e;
+        }
 
-        $this->_em->remove($dataflow);
-        $this->_em->flush();
+        $this->connection->commit();
+    }
+
+    public function createQueryBuilder($alias = null): QueryBuilder
+    {
+        $qb = $this->connection->createQueryBuilder();
+        $qb->select('*')
+            ->from(static::TABLE_NAME, $alias);
+
+        return $qb;
+    }
+
+    private function returnFirstOrNull(QueryBuilder $qb): ?ScheduledDataflow
+    {
+        $stmt = $qb->execute();
+        if (0 === $stmt->rowCount()) {
+            return null;
+        }
+
+        return ScheduledDataflow::createFromArray($this->initDateTime($this->initArray($stmt->fetchAssociative())));
     }
 }

src/Resources/config/messenger_services.yaml

@@ -0,0 +1,12 @@
+services:
+    CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunnerInterface: '@CodeRhapsodie\DataflowBundle\Runner\MessengerDataflowRunner'
+    CodeRhapsodie\DataflowBundle\Runner\MessengerDataflowRunner:
+        arguments:
+            $repository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
+            $bus: ~ # Filled in compiler pass
+
+    CodeRhapsodie\DataflowBundle\MessengerMode\JobMessageHandler:
+        arguments:
+            $repository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
+            $processor: '@CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface'
+        tags: ['messenger.message_handler']

src/Resources/config/services.yaml

@@ -10,53 +10,79 @@ services:
             $registry: '@CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface'
             $scheduledDataflowRepository: '@CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository'
             $validator: '@validator'
+            $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
         tags: ['console.command']
 
     CodeRhapsodie\DataflowBundle\Command\ChangeScheduleStatusCommand:
         arguments:
             $scheduledDataflowRepository: '@CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository'
+            $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
         tags: ['console.command']
 
     CodeRhapsodie\DataflowBundle\Command\ExecuteDataflowCommand:
         arguments:
             $registry: '@CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface'
+            $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
         tags: ['console.command']
 
     CodeRhapsodie\DataflowBundle\Command\JobShowCommand:
         arguments:
             $jobRepository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
+            $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
         tags: ['console.command']
 
     CodeRhapsodie\DataflowBundle\Command\RunPendingDataflowsCommand:
         arguments:
             $manager: '@CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManagerInterface'
             $runner: '@CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunnerInterface'
+            $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
         tags: ['console.command']
 
     CodeRhapsodie\DataflowBundle\Command\ScheduleListCommand:
         arguments:
             $scheduledDataflowRepository: '@CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository'
+            $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
+        tags: ['console.command']
+
+    CodeRhapsodie\DataflowBundle\Command\SchemaCommand:
+        arguments:
+            $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
         tags: ['console.command']
 
     CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository:
-        factory: ['@doctrine.orm.default_entity_manager', 'getRepository']
-        arguments: ['CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow']
+        lazy: true
+        arguments: ['@coderhapsodie.dataflow.connection']
 
     CodeRhapsodie\DataflowBundle\Repository\JobRepository:
-        factory: ['@doctrine.orm.default_entity_manager', 'getRepository']
-        arguments: ['CodeRhapsodie\DataflowBundle\Entity\Job']
+        lazy: true
+        arguments: ['@coderhapsodie.dataflow.connection']
+
+    coderhapsodie.dataflow.connection: "@coderhapsodie.dataflow.connection.internal"
+
+    coderhapsodie.dataflow.connection.internal:
+        lazy: true
+        class: Doctrine\DBAL\Connection
+        factory: ['@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory', 'getConnection']
+
+    CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory:
+        arguments: ['@service_container', '%coderhapsodie.dataflow.dbal_default_connection%']
 
     CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManagerInterface: '@CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManager'
     CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManager:
         arguments:
-            $em: '@doctrine.orm.default_entity_manager'
+            $connection: '@coderhapsodie.dataflow.connection'
             $scheduledDataflowRepository: '@CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository'
             $jobRepository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
 
     CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunnerInterface: '@CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunner'
     CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunner:
         arguments:
-            $em: '@doctrine.orm.default_entity_manager'
+            $repository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
+            $processor: '@CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface'
+
+    CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface: '@CodeRhapsodie\DataflowBundle\Processor\JobProcessor'
+    CodeRhapsodie\DataflowBundle\Processor\JobProcessor:
+        arguments:
             $repository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
             $registry: '@CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface'
             $dispatcher: '@event_dispatcher'

src/Runner/MessengerDataflowRunner.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\Runner;
+
+use CodeRhapsodie\DataflowBundle\Entity\Job;
+use CodeRhapsodie\DataflowBundle\MessengerMode\JobMessage;
+use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
+use Symfony\Component\Messenger\MessageBusInterface;
+
+class MessengerDataflowRunner implements PendingDataflowRunnerInterface
+{
+    public function __construct(private JobRepository $repository, private MessageBusInterface $bus)
+    {
+    }
+
+    public function runPendingDataflows(): void
+    {
+        while (null !== ($job = $this->repository->findNextPendingDataflow())) {
+            $this->bus->dispatch(new JobMessage($job->getId()));
+            $job->setStatus(Job::STATUS_QUEUED);
+            $this->repository->save($job);
+        }
+    }
+}

src/Runner/PendingDataflowRunner.php

@@ -4,35 +4,13 @@ declare(strict_types=1);
 
 namespace CodeRhapsodie\DataflowBundle\Runner;
 
-use CodeRhapsodie\DataflowBundle\DataflowType\Result;
-use CodeRhapsodie\DataflowBundle\Entity\Job;
-use CodeRhapsodie\DataflowBundle\Event\Events;
-use CodeRhapsodie\DataflowBundle\Event\ProcessingEvent;
-use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface;
+use CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface;
 use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
-use Doctrine\ORM\EntityManagerInterface;
-use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 
 class PendingDataflowRunner implements PendingDataflowRunnerInterface
 {
-    /** @var EntityManagerInterface */
-    private $em;
-
-    /** @var JobRepository */
-    private $repository;
-
-    /** @var DataflowTypeRegistryInterface */
-    private $registry;
-
-    /** @var EventDispatcherInterface */
-    private $dispatcher;
-
-    public function __construct(EntityManagerInterface $em, JobRepository $repository, DataflowTypeRegistryInterface $registry, EventDispatcherInterface $dispatcher)
+    public function __construct(private JobRepository $repository, private JobProcessorInterface $processor)
     {
-        $this->em = $em;
-        $this->repository = $repository;
-        $this->registry = $registry;
-        $this->dispatcher = $dispatcher;
     }
 
     /**
@@ -41,49 +19,7 @@ class PendingDataflowRunner implements PendingDataflowRunnerInterface
     public function runPendingDataflows(): void
     {
         while (null !== ($job = $this->repository->findNextPendingDataflow())) {
-            $this->beforeProcessing($job);
-
-            $dataflowType = $this->registry->getDataflowType($job->getDataflowType());
-            $result = $dataflowType->process($job->getOptions());
-
-            $this->afterProcessing($job, $result);
+            $this->processor->process($job);
         }
     }
-
-    /**
-     * @param Job $job
-     */
-    private function beforeProcessing(Job $job): void
-    {
-        $this->dispatcher->dispatch(Events::BEFORE_PROCESSING, new ProcessingEvent($job));
-
-        $job
-            ->setStatus(Job::STATUS_RUNNING)
-            ->setStartTime(new \DateTime())
-        ;
-        $this->em->flush();
-    }
-
-    /**
-     * @param Job    $job
-     * @param Result $result
-     */
-    private function afterProcessing(Job $job, Result $result): void
-    {
-        $exceptions = [];
-        /** @var \Exception $exception */
-        foreach ($result->getExceptions() as $exception) {
-            $exceptions[] = (string) $exception;
-        }
-
-        $job
-            ->setEndTime($result->getEndTime())
-            ->setStatus(Job::STATUS_COMPLETED)
-            ->setCount($result->getSuccessCount())
-            ->setExceptions($exceptions)
-        ;
-        $this->em->flush();
-
-        $this->dispatcher->dispatch(Events::AFTER_PROCESSING, new ProcessingEvent($job));
-    }
 }

src/SchemaProvider/DataflowSchemaProvider.php

@@ -0,0 +1,55 @@
+<?php
+
+declare(strict_types=1);
+
+namespace CodeRhapsodie\DataflowBundle\SchemaProvider;
+
+use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
+use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
+use Doctrine\DBAL\Schema\Schema;
+
+/**
+ * Class JobSchemaProvider.
+ *
+ * @codeCoverageIgnore
+ */
+class DataflowSchemaProvider
+{
+    public function createSchema()
+    {
+        $schema = new Schema();
+        $tableJob = $schema->createTable(JobRepository::TABLE_NAME);
+        $tableJob->addColumn('id', 'integer', [
+            'autoincrement' => true,
+        ]);
+        $tableJob->setPrimaryKey(['id']);
+
+        $tableJob->addColumn('scheduled_dataflow_id', 'integer', ['notnull' => false]);
+        $tableJob->addColumn('status', 'integer', ['notnull' => true]);
+        $tableJob->addColumn('label', 'string', ['notnull' => true, 'length' => 255]);
+        $tableJob->addColumn('dataflow_type', 'string', ['notnull' => true, 'length' => 255]);
+        $tableJob->addColumn('options', 'json', ['notnull' => true]);
+        $tableJob->addColumn('requested_date', 'datetime', ['notnull' => false]);
+        $tableJob->addColumn('count', 'integer', ['notnull' => false]);
+        $tableJob->addColumn('exceptions', 'json', ['notnull' => false]);
+        $tableJob->addColumn('start_time', 'datetime', ['notnull' => false]);
+        $tableJob->addColumn('end_time', 'datetime', ['notnull' => false]);
+
+        $tableSchedule = $schema->createTable(ScheduledDataflowRepository::TABLE_NAME);
+        $tableSchedule->addColumn('id', 'integer', [
+            'autoincrement' => true,
+        ]);
+        $tableSchedule->setPrimaryKey(['id']);
+        $tableSchedule->addColumn('label', 'string', ['notnull' => true, 'length' => 255]);
+        $tableSchedule->addColumn('dataflow_type', 'string', ['notnull' => true, 'length' => 255]);
+        $tableSchedule->addColumn('options', 'json', ['notnull' => true]);
+        $tableSchedule->addColumn('frequency', 'string', ['notnull' => true, 'length' => 255]);
+        $tableSchedule->addColumn('next', 'datetime', ['notnull' => false]);
+        $tableSchedule->addColumn('enabled', 'boolean', ['notnull' => true]);
+
+        $tableJob->addForeignKeyConstraint($tableSchedule, ['scheduled_dataflow_id'], ['id']);
+        $tableJob->addIndex(['status'], 'idx_status');
+
+        return $schema;
+    }
+}

src/Validator/Constraints/Frequency.php

@@ -8,10 +8,10 @@ use Symfony\Component\Validator\Constraint;
 
 /**
  * @Annotation
+ *
+ * @codeCoverageIgnore
  */
 class Frequency extends Constraint
 {
-    public $invalidMessage = 'The provided frequency "{{ string }}" must be a valid parameter for DateInterval::createFromDateString()';
-
-    public $negativeIntervalMessage = 'The provided frequency "{{ string }}" mustn\'t represent a negative interval';
+    public $message = 'The provided frequency "{{ string }}" must be a valid parameter for DateInterval::createFromDateString() and must not represent a negative value';
 }

src/Validator/Constraints/FrequencyValidator.php

@@ -25,7 +25,7 @@ class FrequencyValidator extends ConstraintValidator
 
         $interval = @\DateInterval::createFromDateString($value);
         if (!$interval) {
-            $this->context->buildViolation($constraint->invalidMessage)
+            $this->context->buildViolation($constraint->message)
                 ->setParameter('{{ string }}', $value)
                 ->addViolation()
             ;
@@ -38,7 +38,7 @@ class FrequencyValidator extends ConstraintValidator
         $dt->add($interval);
 
         if ($dt <= $now) {
-            $this->context->buildViolation($constraint->negativeIntervalMessage)
+            $this->context->buildViolation($constraint->message)
                 ->setParameter('{{ string }}', $value)
                 ->addViolation()
             ;