mirror of
https://github.com/code-rhapsodie/dataflow-bundle.git
synced 2026-03-24 06:42:23 +01:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4c80b3c51b | ||
|
|
23801f8785 | ||
|
|
689d79535c | ||
|
|
f17322bdd8 | ||
|
|
90855562ca | ||
|
|
776323118a | ||
|
|
8371e2d1a6 | ||
|
|
426c7b4324 | ||
|
|
567a70aa84 |
3
.coveralls.yml
Normal file
3
.coveralls.yml
Normal file
@@ -0,0 +1,3 @@
|
||||
service_name : travis-ci
|
||||
coverage_clover: var/build/clover.xml
|
||||
json_path : var/build/upload.json
|
||||
111
.travis.yml
Normal file
111
.travis.yml
Normal file
@@ -0,0 +1,111 @@
|
||||
language: php
|
||||
|
||||
sudo: false
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.composer/cache
|
||||
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- /^\d+\.\d+$/
|
||||
- travis-setup
|
||||
|
||||
env:
|
||||
global:
|
||||
- SYMFONY_DEPRECATIONS_HELPER="max[self]=0"
|
||||
- PHPUNIT_FLAGS="-v"
|
||||
- PHPUNIT_ENABLED="true"
|
||||
- STABILITY=stable
|
||||
- COVERALLS_ENABLED="false"
|
||||
|
||||
matrix:
|
||||
fast_finish: true
|
||||
include:
|
||||
- php: '7.1'
|
||||
- php: '7.2'
|
||||
- php: '7.3'
|
||||
|
||||
# Enable code coverage with the latest supported PHP version
|
||||
- php: '7.3'
|
||||
env:
|
||||
- COVERALLS_ENABLED="true"
|
||||
- PHPUNIT_FLAGS="-v --coverage-text --coverage-clover var/build/clover.xml"
|
||||
|
||||
# Minimum supported dependencies with the latest and oldest supported PHP versions
|
||||
- php: '7.1'
|
||||
env:
|
||||
- COMPOSER_FLAGS="--prefer-lowest"
|
||||
- php: '7.3'
|
||||
env:
|
||||
- COMPOSER_FLAGS="--prefer-lowest"
|
||||
|
||||
# Test each supported Symfony version with lowest supported PHP version
|
||||
- php: '7.1'
|
||||
env:
|
||||
- SYMFONY_VERSION=3.4.*
|
||||
- php: '7.1'
|
||||
env:
|
||||
- SYMFONY_VERSION=4.2.*
|
||||
- php: '7.1'
|
||||
env:
|
||||
- SYMFONY_VERSION=4.3.*
|
||||
|
||||
# Test unsupported versions of Symfony
|
||||
- php: '7.1'
|
||||
env:
|
||||
- SYMFONY_VERSION=4.0.*
|
||||
- php: '7.1'
|
||||
env:
|
||||
- SYMFONY_VERSION=4.1.*
|
||||
|
||||
|
||||
# Test upcoming Symfony versions with lowest supported PHP version and dev dependencies
|
||||
- php: '7.1'
|
||||
env:
|
||||
- STABILITY=dev
|
||||
- SYMFONY_VERSION=4.4.*
|
||||
|
||||
# Test upcoming PHP versions with dev dependencies
|
||||
- php: '7.4snapshot'
|
||||
env:
|
||||
- STABILITY=dev
|
||||
- COMPOSER_FLAGS="--ignore-platform-reqs --prefer-stable"
|
||||
|
||||
allow_failures:
|
||||
- env:
|
||||
- SYMFONY_VERSION=4.0.*
|
||||
- env:
|
||||
- SYMFONY_VERSION=4.1.*
|
||||
- env:
|
||||
- STABILITY=dev
|
||||
- COMPOSER_FLAGS="--ignore-platform-reqs --prefer-stable"
|
||||
- env:
|
||||
- STABILITY=dev
|
||||
- SYMFONY_VERSION=4.4.*
|
||||
|
||||
before_install:
|
||||
- if [[ "$SYMFONY_VERSION" != "" ]]; then
|
||||
travis_retry composer global require "symfony/flex:^1.4";
|
||||
composer config extra.symfony.require $SYMFONY_VERSION;
|
||||
fi
|
||||
- if [[ "$STABILITY" != "stable" ]]; then
|
||||
travis_retry composer config minimum-stability $STABILITY;
|
||||
fi
|
||||
- if [[ "$COVERALLS_ENABLED" != "true" ]]; then
|
||||
phpenv config-rm xdebug.ini || true;
|
||||
fi
|
||||
- if [[ "$COVERALLS_ENABLED" == "true" ]]; then
|
||||
travis_retry composer require --dev satooshi/php-coveralls:^2.0 --no-update $COMPOSER_FLAGS;
|
||||
fi
|
||||
|
||||
install:
|
||||
- travis_retry composer update --prefer-dist --no-interaction --no-suggest --no-progress --ansi $COMPOSER_FLAGS
|
||||
|
||||
script: ./vendor/bin/phpunit $PHPUNIT_FLAGS
|
||||
|
||||
after_success:
|
||||
- if [[ "$PHPUNIT_ENABLED" == "true" && "$COVERALLS_ENABLED" == "true" ]]; then
|
||||
./vendor/bin/php-coveralls -vvv --config .coveralls.yml;
|
||||
fi;
|
||||
91
README.md
91
README.md
@@ -1,8 +1,26 @@
|
||||
# Code-Rhapsodie Dataflow Bundle
|
||||
# Code Rhapsodie Dataflow Bundle
|
||||
|
||||
DataflowBundle is a bundle for Symfony 3.4+
|
||||
providing an easy way to create import / export dataflow.
|
||||
|
||||
[](https://travis-ci.org/code-rhapsodie/dataflow-bundle)
|
||||
|
||||
[](https://coveralls.io/github/code-rhapsodie/dataflow-bundle)
|
||||
|
||||
Dataflow uses a linear generic workflow in three parts:
|
||||
* one reader
|
||||
* any number of steps
|
||||
* one or more writers
|
||||
|
||||
The reader can read data from anywhere and return data row by row. Each step processes the current row data.
|
||||
The steps are executed in the order in which they are added.
|
||||
And, one or more writers save the row anywhere you want.
|
||||
|
||||
As the following schema shows, you can define more than one dataflow:
|
||||
|
||||

|
||||
|
||||
|
||||
# Features
|
||||
|
||||
* Define and configure a Dataflow
|
||||
@@ -80,12 +98,12 @@ public function registerBundles()
|
||||
|
||||
### Update the database
|
||||
|
||||
This bundle use Dotrine ORM for drive the database table for store Dataflow schedule (`cr_dataflow_scheduled`)
|
||||
This bundle uses Doctrine ORM for drive the database table for store Dataflow schedule (`cr_dataflow_scheduled`)
|
||||
and jobs (`cr_dataflow_job`).
|
||||
|
||||
#### Doctrine migration
|
||||
|
||||
Execute the command for generate the migration for your database:
|
||||
Execute the command to generate the migration for your database:
|
||||
|
||||
```shell script
|
||||
$ bin/console doctrine:migration:diff
|
||||
@@ -93,7 +111,7 @@ $ bin/console doctrine:migration:diff
|
||||
|
||||
#### Other migration tools
|
||||
|
||||
If you use [Phinx](https://phinx.org/) or [Kaliop Migration Bundle](https://github.com/kaliop-uk/ezmigrationbundle) or whatever,
|
||||
If you use [Phinx](https://phinx.org/) or [Kaliop Migration Bundle](https://github.com/kaliop-uk/ezmigrationbundle) or whatever,
|
||||
you can add a new migration with the generated SQL query from this command:
|
||||
|
||||
```shell script
|
||||
@@ -105,17 +123,17 @@ $ bin/console doctrine:schema:update --dump-sql
|
||||
|
||||
This bundle uses a fixed and simple workflow structure in order to let you focus on the data processing logic part of your dataflow.
|
||||
|
||||
A dataflow type defines the different parts of your dataflow. A dataflow is comprised of:
|
||||
A dataflow type defines the different parts of your dataflow. A dataflow is made of:
|
||||
- exactly one *Reader*
|
||||
- any number of *Steps*
|
||||
- one or more *Writers*
|
||||
|
||||
Dataflow types can be configured with options.
|
||||
|
||||
A dataflow type must implements `CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface`.
|
||||
A dataflow type must implement `CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface`.
|
||||
|
||||
To help with creating your workflow types, an abstract class `CodeRhapsodie\DataflowBundle\DataflowType\AbstractDataflowType`
|
||||
is provided, allowing you to define your dataflow through an handy builder `CodeRhapsodie\DataflowBundle\DataflowType\DataflowBuilder`.
|
||||
To help with creating your dataflow types, an abstract class `CodeRhapsodie\DataflowBundle\DataflowType\AbstractDataflowType`
|
||||
is provided, allowing you to define your dataflow through a handy builder `CodeRhapsodie\DataflowBundle\DataflowType\DataflowBuilder`.
|
||||
|
||||
This is an example to define one class DataflowType:
|
||||
|
||||
@@ -176,8 +194,11 @@ class MyFirstDataflowType extends AbstractDataflowType
|
||||
|
||||
```
|
||||
|
||||
The `DataflowTypeInterface` is used by Symfony for auto-configuration our custom datafow type only if the folder is correctly configured (see the `services` configuration file in your projet).
|
||||
If you don't use the auto-configuration, you must add this tag `coderhapsodie.dataflow.type` in your dataflow type service configuration:
|
||||
Dataflow types must be tagged with `coderhapsodie.dataflow.type`.
|
||||
|
||||
If you're using Symfony auto-configuration for your services, this tag will be automatically added to all services implementing `DataflowTypeInterface`.
|
||||
|
||||
Otherwise, manually add the tag `coderhapsodie.dataflow.type` in your dataflow type service configuration:
|
||||
|
||||
```yaml
|
||||
CodeRhapsodie\DataflowExemple\DataflowType\MyFirstDataflowType:
|
||||
@@ -185,9 +206,9 @@ If you don't use the auto-configuration, you must add this tag `coderhapsodie.da
|
||||
- { name: coderhapsodie.dataflow.type }
|
||||
```
|
||||
|
||||
### Use the options for your dataflow type
|
||||
### Use options for your dataflow type
|
||||
|
||||
The `AbstractDataflowType` can help you define the options of your Datataflow type.
|
||||
The `AbstractDataflowType` can help you define options for your Dataflow type.
|
||||
|
||||
Add this method in your DataflowType class:
|
||||
|
||||
@@ -213,7 +234,6 @@ class MyFirstDataflowType extends AbstractDataflowType
|
||||
|
||||
With this configuration, the option `fileName` is required. For an advanced usage of the option resolver, read the [Symfony documentation](https://symfony.com/doc/current/components/options_resolver.html).
|
||||
|
||||
|
||||
### Check if your DataflowType is ready
|
||||
|
||||
Execute this command to check if your DataflowType is correctly registered:
|
||||
@@ -239,9 +259,9 @@ Symfony Container Public and Private Services Tagged with "coderhapsodie.dataflo
|
||||
|
||||
### Readers
|
||||
|
||||
*Readers* provide the workflow with elements to import / export. Usually, elements are read from an external resource (file, database, webservice, etc).
|
||||
*Readers* provide the dataflow with elements to import / export. Usually, elements are read from an external resource (file, database, webservice, etc).
|
||||
|
||||
A *Reader* must implements `Port\Reader` or return a `iterable` if you use the `Port\Reader\IteratorReader`.
|
||||
A *Reader* can be any `iterable`.
|
||||
|
||||
The only constraint on the returned elements typing is that they cannot be `false`.
|
||||
|
||||
@@ -280,10 +300,10 @@ class FileReader
|
||||
}
|
||||
```
|
||||
|
||||
To setup your reader in the dataflow builder, you must use `Port\Reader\IteratorReader` like this
|
||||
You can set up this reader as follows:
|
||||
|
||||
```php
|
||||
$builder->setReader(new \Port\Reader\IteratorReader($this->myReader))
|
||||
$builder->setReader(($this->myReader)())
|
||||
```
|
||||
|
||||
|
||||
@@ -291,18 +311,37 @@ $builder->setReader(new \Port\Reader\IteratorReader($this->myReader))
|
||||
|
||||
*Steps* are operations performed on the elements before they are handled by the *Writers*. Usually, steps are either:
|
||||
- converters, that alter the element
|
||||
- filters, that conditionally prevents further operations on the element
|
||||
- filters, that conditionally prevent further operations on the element
|
||||
|
||||
A *Step* can be any callable, taking the element as its argument, and returning either:
|
||||
- the element, possibly altered
|
||||
- `false`, if no further operations should be performed on this element
|
||||
|
||||
A few examples:
|
||||
|
||||
```php
|
||||
$builder->addStep(function($item) {
|
||||
// Titles are changed to all caps before export
|
||||
$item['title'] = strtoupper($item['title']);
|
||||
|
||||
return $item;
|
||||
});
|
||||
|
||||
$builder->addStep(function($item) {
|
||||
// Private items are not exported
|
||||
if ($item['private']) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return $item;
|
||||
});
|
||||
```
|
||||
|
||||
### Writers
|
||||
|
||||
*Writers* performs the actual import / export operations.
|
||||
*Writers* perform the actual import / export operations.
|
||||
|
||||
A *Writer* must implements `CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface`.
|
||||
A *Writer* must implement `CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface`.
|
||||
As this interface is not compatible with `Port\Writer`, the adapter `CodeRhapsodie\DataflowBundle\DataflowType\Writer\PortWriterAdapter` is provided.
|
||||
|
||||
This example show how to use the predefined PhpPort Writer :
|
||||
@@ -311,7 +350,7 @@ This example show how to use the predefined PhpPort Writer :
|
||||
$builder->addWriter(new PortWriterAdapter(new \Port\FileWriter()));
|
||||
```
|
||||
|
||||
Or you own Writer:
|
||||
Or your own Writer:
|
||||
|
||||
```php
|
||||
<?php
|
||||
@@ -347,17 +386,17 @@ class FileWriter implements WriterInterface
|
||||
|
||||
All pending dataflow job processes are stored in a queue into the database.
|
||||
|
||||
Add this command into your crontab for execute all queued job:
|
||||
Add this command into your crontab for execute all queued jobs:
|
||||
|
||||
```shell script
|
||||
$ SYMFONY_ENV=prod php bin/console code-rhapsodie:dataflow:job:run-pending
|
||||
$ SYMFONY_ENV=prod php bin/console code-rhapsodie:dataflow:run-pending
|
||||
```
|
||||
|
||||
## Commands
|
||||
|
||||
Many commands are provided.
|
||||
Several commands are provided to manage schedules and run jobs.
|
||||
|
||||
`code-rhapsodie:dataflow:job:run-pending` Executes job in the queue according to their schedule.
|
||||
`code-rhapsodie:dataflow:run-pending` Executes job in the queue according to their schedule.
|
||||
|
||||
`code-rhapsodie:dataflow:schedule:list` Display the list of dataflows scheduled.
|
||||
|
||||
@@ -367,7 +406,7 @@ Many commands are provided.
|
||||
|
||||
`code-rhapsodie:dataflow:job:show` Display the last result of a job.
|
||||
|
||||
`code-rhapsodie:dataflow:execute` Lets you execute one dataflow job.
|
||||
`code-rhapsodie:dataflow:execute` Let you execute one dataflow job.
|
||||
|
||||
|
||||
# Issues and feature requests
|
||||
|
||||
53
Tests/DataflowType/AbstractDataflowTypeTest.php
Normal file
53
Tests/DataflowType/AbstractDataflowTypeTest.php
Normal file
@@ -0,0 +1,53 @@
|
||||
<?php
|
||||
|
||||
namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType;
|
||||
|
||||
use CodeRhapsodie\DataflowBundle\DataflowType\AbstractDataflowType;
|
||||
use CodeRhapsodie\DataflowBundle\DataflowType\DataflowBuilder;
|
||||
use PHPUnit\Framework\Constraint\IsIdentical;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Symfony\Component\OptionsResolver\OptionsResolver;
|
||||
|
||||
class AbstractDataflowTypeTest extends TestCase
|
||||
{
|
||||
public function testProcess()
|
||||
{
|
||||
$label = 'Test label';
|
||||
$options = ['testOption' => 'Test value'];
|
||||
$values = [1, 2, 3];
|
||||
|
||||
$dataflowType = new class($label, $options, $values) extends AbstractDataflowType
|
||||
{
|
||||
private $label;
|
||||
private $options;
|
||||
private $values;
|
||||
|
||||
public function __construct(string $label, array $options, array $values)
|
||||
{
|
||||
$this->label = $label;
|
||||
$this->options = $options;
|
||||
$this->values = $values;
|
||||
}
|
||||
|
||||
public function getLabel(): string
|
||||
{
|
||||
return $this->label;
|
||||
}
|
||||
|
||||
protected function configureOptions(OptionsResolver $optionsResolver): void
|
||||
{
|
||||
$optionsResolver->setDefined('testOption');
|
||||
}
|
||||
|
||||
protected function buildDataflow(DataflowBuilder $builder, array $options): void
|
||||
{
|
||||
$builder->setReader($this->values);
|
||||
(new IsIdentical($this->options))->evaluate($options);
|
||||
}
|
||||
};
|
||||
|
||||
$result = $dataflowType->process($options);
|
||||
$this->assertSame($label, $result->getName());
|
||||
$this->assertSame(count($values), $result->getTotalProcessedCount());
|
||||
}
|
||||
}
|
||||
37
Tests/DataflowType/Writer/PortWriterAdapterTest.php
Normal file
37
Tests/DataflowType/Writer/PortWriterAdapterTest.php
Normal file
@@ -0,0 +1,37 @@
|
||||
<?php
|
||||
|
||||
namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType\Writer;
|
||||
|
||||
use CodeRhapsodie\DataflowBundle\DataflowType\Writer\PortWriterAdapter;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class PortWriterAdapterTest extends TestCase
|
||||
{
|
||||
public function testAll()
|
||||
{
|
||||
$value = 'not an array';
|
||||
|
||||
$writer = $this->getMockBuilder('\Port\Writer')
|
||||
->setMethods(['prepare', 'finish', 'writeItem'])
|
||||
->getMock()
|
||||
;
|
||||
$writer
|
||||
->expects($this->once())
|
||||
->method('prepare')
|
||||
;
|
||||
$writer
|
||||
->expects($this->once())
|
||||
->method('finish')
|
||||
;
|
||||
$writer
|
||||
->expects($this->once())
|
||||
->method('writeItem')
|
||||
->with([$value])
|
||||
;
|
||||
|
||||
$adapter = new PortWriterAdapter($writer);
|
||||
$adapter->prepare();
|
||||
$adapter->write($value);
|
||||
$adapter->finish();
|
||||
}
|
||||
}
|
||||
@@ -103,10 +103,8 @@ class PendingDataflowRunnerTest extends TestCase
|
||||
->willReturnOnConsecutiveCalls($dataflowType1, $dataflowType2)
|
||||
;
|
||||
|
||||
$bag1 = new \SplObjectStorage();
|
||||
$bag1->attach(new \Exception('message1'));
|
||||
$bag2 = new \SplObjectStorage();
|
||||
$bag2->attach(new \Exception('message2'));
|
||||
$bag1 = [new \Exception('message1')];
|
||||
$bag2 = [new \Exception('message2')];
|
||||
|
||||
$result1 = new Result('name', new \DateTime(), $end1 = new \DateTime(), $count1 = 10, $bag1);
|
||||
$result2 = new Result('name', new \DateTime(), $end2 = new \DateTime(), $count2 = 20, $bag2);
|
||||
|
||||
@@ -36,7 +36,7 @@ class FrequencyValidatorTest extends ConstraintValidatorTestCase
|
||||
public function testInvalidValue()
|
||||
{
|
||||
$constraint = new Frequency([
|
||||
'invalidMessage' => 'testMessage',
|
||||
'message' => 'testMessage',
|
||||
]);
|
||||
|
||||
$this->validator->validate('wrong value', $constraint);
|
||||
@@ -53,7 +53,7 @@ class FrequencyValidatorTest extends ConstraintValidatorTestCase
|
||||
public function testNegativeIntervals($value)
|
||||
{
|
||||
$constraint = new Frequency([
|
||||
'negativeIntervalMessage' => 'testMessage',
|
||||
'message' => 'testMessage',
|
||||
]);
|
||||
|
||||
$this->validator->validate($value, $constraint);
|
||||
|
||||
@@ -37,21 +37,32 @@
|
||||
},
|
||||
"require": {
|
||||
"php": "^7.1",
|
||||
"symfony/dependency-injection": "^3.4||^4.0",
|
||||
"symfony/http-kernel": "^3.4||^4.0",
|
||||
"doctrine/orm": "^2.4.5",
|
||||
"seld/signal-handler": "^1.0",
|
||||
"symfony/config": "^3.4||^4.0",
|
||||
"symfony/yaml": "^3.4||^4.0",
|
||||
"symfony/console": "^3.4||^4.0",
|
||||
"symfony/lock": "^3.4||^4.0",
|
||||
"symfony/dependency-injection": "^3.4||^4.0",
|
||||
"symfony/event-dispatcher": "^3.4||^4.0",
|
||||
"symfony/validator": "^3.4||^4.0",
|
||||
"symfony/http-kernel": "^3.4||^4.0",
|
||||
"symfony/lock": "^3.4||^4.0",
|
||||
"symfony/options-resolver": "^3.4||^4.0",
|
||||
"doctrine/orm": "^2.4.5"
|
||||
"symfony/validator": "^3.4||^4.0",
|
||||
"symfony/yaml": "^3.4||^4.0",
|
||||
"doctrine/doctrine-bundle": "^1.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^8.1"
|
||||
"friendsofphp/php-cs-fixer": "^2.15",
|
||||
"phpunit/phpunit": "^7||^8"
|
||||
},
|
||||
"suggest": {
|
||||
"portphp/portphp": "Provides generic readers, steps and writers for your dataflows."
|
||||
},
|
||||
"config": {
|
||||
"sort-packages": true
|
||||
},
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "1.x-dev"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
<ini name="error_reporting" value="-1" />
|
||||
</php>
|
||||
<testsuites>
|
||||
<testsuite name="Port tests suite">
|
||||
<testsuite name="Dataflow tests suite">
|
||||
<directory suffix="Test.php">./Tests</directory>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
|
||||
@@ -9,6 +9,9 @@ use CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler\DataflowTypeCompil
|
||||
use Symfony\Component\DependencyInjection\ContainerBuilder;
|
||||
use Symfony\Component\HttpKernel\Bundle\Bundle;
|
||||
|
||||
/**
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class CodeRhapsodieDataflowBundle extends Bundle
|
||||
{
|
||||
protected $name = 'CodeRhapsodieDataflowBundle';
|
||||
|
||||
@@ -14,6 +14,9 @@ use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
use Symfony\Component\Validator\Validator\ValidatorInterface;
|
||||
|
||||
/**
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class AddScheduledDataflowCommand extends Command
|
||||
{
|
||||
protected static $defaultName = 'code-rhapsodie:dataflow:schedule:add';
|
||||
|
||||
@@ -13,6 +13,9 @@ use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
|
||||
/**
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class ChangeScheduleStatusCommand extends Command
|
||||
{
|
||||
protected static $defaultName = 'code-rhapsodie:dataflow:schedule:change-status';
|
||||
|
||||
@@ -12,6 +12,8 @@ use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
/**
|
||||
* Runs one dataflow.
|
||||
*
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class ExecuteDataflowCommand extends Command
|
||||
{
|
||||
|
||||
@@ -12,6 +12,9 @@ use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
|
||||
/**
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class JobShowCommand extends Command
|
||||
{
|
||||
private const STATUS_MAPPING = [
|
||||
|
||||
@@ -13,6 +13,8 @@ use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
/**
|
||||
* Runs dataflows according to user-defined schedule.
|
||||
*
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class RunPendingDataflowsCommand extends Command
|
||||
{
|
||||
|
||||
@@ -10,6 +10,9 @@ use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
|
||||
/**
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class ScheduleListCommand extends Command
|
||||
{
|
||||
protected static $defaultName = 'code-rhapsodie:dataflow:schedule:list';
|
||||
|
||||
@@ -8,6 +8,9 @@ use Symfony\Component\OptionsResolver\OptionsResolver;
|
||||
|
||||
abstract class AbstractDataflowType implements DataflowTypeInterface
|
||||
{
|
||||
/**
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
public function getAliases(): iterable
|
||||
{
|
||||
return [];
|
||||
@@ -28,6 +31,9 @@ abstract class AbstractDataflowType implements DataflowTypeInterface
|
||||
return $dataflow->process();
|
||||
}
|
||||
|
||||
/**
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
protected function configureOptions(OptionsResolver $optionsResolver): void
|
||||
{
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ namespace CodeRhapsodie\DataflowBundle\DataflowType\Dataflow;
|
||||
use CodeRhapsodie\DataflowBundle\DataflowType\Result;
|
||||
use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
|
||||
use CodeRhapsodie\DataflowBundle\Exceptions\InterruptedProcessingException;
|
||||
use Seld\Signal\SignalHandler;
|
||||
|
||||
class Dataflow implements DataflowInterface
|
||||
{
|
||||
@@ -25,7 +24,7 @@ class Dataflow implements DataflowInterface
|
||||
|
||||
/**
|
||||
* @param iterable $reader
|
||||
* @param null|string $name
|
||||
* @param string|null $name
|
||||
*/
|
||||
public function __construct(iterable $reader, ?string $name)
|
||||
{
|
||||
@@ -63,13 +62,9 @@ class Dataflow implements DataflowInterface
|
||||
public function process(): Result
|
||||
{
|
||||
$count = 0;
|
||||
$exceptions = new \SplObjectStorage();
|
||||
$exceptions = [];
|
||||
$startTime = new \DateTime();
|
||||
|
||||
SignalHandler::create(['SIGTERM', 'SIGINT'], function () {
|
||||
throw new InterruptedProcessingException();
|
||||
});
|
||||
|
||||
foreach ($this->writers as $writer) {
|
||||
$writer->prepare();
|
||||
}
|
||||
@@ -78,7 +73,7 @@ class Dataflow implements DataflowInterface
|
||||
try {
|
||||
$this->processItem($item);
|
||||
} catch (\Exception $e) {
|
||||
$exceptions->attach($e, $index);
|
||||
$exceptions[$index] = $e;
|
||||
}
|
||||
|
||||
++$count;
|
||||
|
||||
@@ -4,6 +4,9 @@ declare(strict_types=1);
|
||||
|
||||
namespace CodeRhapsodie\DataflowBundle\DataflowType;
|
||||
|
||||
/**
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class Result
|
||||
{
|
||||
/** @var string */
|
||||
@@ -27,7 +30,7 @@ class Result
|
||||
/** @var int */
|
||||
private $totalProcessedCount = 0;
|
||||
|
||||
/** @var \SplObjectStorage */
|
||||
/** @var array */
|
||||
private $exceptions;
|
||||
|
||||
/**
|
||||
@@ -37,7 +40,7 @@ class Result
|
||||
* @param int $totalCount
|
||||
* @param \SplObjectStorage $exceptions
|
||||
*/
|
||||
public function __construct(string $name, \DateTimeInterface $startTime, \DateTimeInterface $endTime, int $totalCount, \SplObjectStorage $exceptions)
|
||||
public function __construct(string $name, \DateTimeInterface $startTime, \DateTimeInterface $endTime, int $totalCount, array $exceptions)
|
||||
{
|
||||
$this->name = $name;
|
||||
$this->startTime = $startTime;
|
||||
@@ -114,9 +117,9 @@ class Result
|
||||
}
|
||||
|
||||
/**
|
||||
* @return \SplObjectStorage
|
||||
* @return array
|
||||
*/
|
||||
public function getExceptions(): \SplObjectStorage
|
||||
public function getExceptions(): array
|
||||
{
|
||||
return $this->exceptions;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,9 @@ use Symfony\Component\DependencyInjection\ContainerBuilder;
|
||||
use Symfony\Component\DependencyInjection\Extension\Extension;
|
||||
use Symfony\Component\DependencyInjection\Loader\YamlFileLoader;
|
||||
|
||||
/**
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class CodeRhapsodieDataflowExtension extends Extension
|
||||
{
|
||||
public function load(array $configs, ContainerBuilder $container)
|
||||
|
||||
@@ -11,6 +11,8 @@ use Symfony\Component\DependencyInjection\Reference;
|
||||
|
||||
/**
|
||||
* Registers dataflow types in the registry.
|
||||
*
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class DataflowTypeCompilerPass implements CompilerPassInterface
|
||||
{
|
||||
|
||||
@@ -11,6 +11,8 @@ use Doctrine\ORM\Mapping as ORM;
|
||||
*
|
||||
* @ORM\Entity(repositoryClass="CodeRhapsodie\DataflowBundle\Repository\JobRepository")
|
||||
* @ORM\Table(name="cr_dataflow_job")
|
||||
*
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class Job
|
||||
{
|
||||
@@ -144,7 +146,7 @@ class Job
|
||||
}
|
||||
|
||||
/**
|
||||
* @return null|string
|
||||
* @return string|null
|
||||
*/
|
||||
public function getLabel(): ?string
|
||||
{
|
||||
@@ -152,7 +154,7 @@ class Job
|
||||
}
|
||||
|
||||
/**
|
||||
* @param null|string $label
|
||||
* @param string|null $label
|
||||
*
|
||||
* @return Job
|
||||
*/
|
||||
@@ -164,7 +166,7 @@ class Job
|
||||
}
|
||||
|
||||
/**
|
||||
* @return null|string
|
||||
* @return string|null
|
||||
*/
|
||||
public function getDataflowType(): ?string
|
||||
{
|
||||
@@ -172,7 +174,7 @@ class Job
|
||||
}
|
||||
|
||||
/**
|
||||
* @param null|string $dataflowType
|
||||
* @param string|null $dataflowType
|
||||
*
|
||||
* @return Job
|
||||
*/
|
||||
|
||||
@@ -12,6 +12,8 @@ use Doctrine\ORM\Mapping as ORM;
|
||||
*
|
||||
* @ORM\Entity(repositoryClass="CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository")
|
||||
* @ORM\Table(name="cr_dataflow_scheduled")
|
||||
*
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class ScheduledDataflow
|
||||
{
|
||||
@@ -78,7 +80,7 @@ class ScheduledDataflow
|
||||
/**
|
||||
* @var Job[]
|
||||
*
|
||||
* @ORM\OneToMany(targetEntity="Job", mappedBy="scheduledDataflow", cascade={"persist"})
|
||||
* @ORM\OneToMany(targetEntity="Job", mappedBy="scheduledDataflow", cascade={"persist", "remove"})
|
||||
* @ORM\OrderBy({"startTime" = "DESC"})
|
||||
*/
|
||||
private $jobs;
|
||||
@@ -92,7 +94,7 @@ class ScheduledDataflow
|
||||
}
|
||||
|
||||
/**
|
||||
* @return null|string
|
||||
* @return string|null
|
||||
*/
|
||||
public function getLabel(): ?string
|
||||
{
|
||||
@@ -100,7 +102,7 @@ class ScheduledDataflow
|
||||
}
|
||||
|
||||
/**
|
||||
* @param null|string $label
|
||||
* @param string|null $label
|
||||
*
|
||||
* @return ScheduledDataflow
|
||||
*/
|
||||
@@ -112,7 +114,7 @@ class ScheduledDataflow
|
||||
}
|
||||
|
||||
/**
|
||||
* @return null|string
|
||||
* @return string|null
|
||||
*/
|
||||
public function getDataflowType(): ?string
|
||||
{
|
||||
@@ -120,7 +122,7 @@ class ScheduledDataflow
|
||||
}
|
||||
|
||||
/**
|
||||
* @param null|string $dataflowType
|
||||
* @param string|null $dataflowType
|
||||
*
|
||||
* @return ScheduledDataflow
|
||||
*/
|
||||
|
||||
@@ -9,6 +9,8 @@ use Symfony\Component\EventDispatcher\Event;
|
||||
|
||||
/**
|
||||
* Event used during the dataflow lifecycle.
|
||||
*
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class ProcessingEvent extends Event
|
||||
{
|
||||
|
||||
@@ -11,6 +11,8 @@ use Doctrine\ORM\EntityRepository;
|
||||
|
||||
/**
|
||||
* Repository.
|
||||
*
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class JobRepository extends EntityRepository
|
||||
{
|
||||
|
||||
@@ -10,6 +10,8 @@ use Doctrine\ORM\EntityRepository;
|
||||
|
||||
/**
|
||||
* Repository for the ScheduledDataflow entity.
|
||||
*
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class ScheduledDataflowRepository extends EntityRepository
|
||||
{
|
||||
|
||||
BIN
src/Resources/doc/schema.png
Normal file
BIN
src/Resources/doc/schema.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 130 KiB |
@@ -8,10 +8,10 @@ use Symfony\Component\Validator\Constraint;
|
||||
|
||||
/**
|
||||
* @Annotation
|
||||
*
|
||||
* @codeCoverageIgnore
|
||||
*/
|
||||
class Frequency extends Constraint
|
||||
{
|
||||
public $invalidMessage = 'The provided frequency "{{ string }}" must be a valid parameter for DateInterval::createFromDateString()';
|
||||
|
||||
public $negativeIntervalMessage = 'The provided frequency "{{ string }}" mustn\'t represent a negative interval';
|
||||
public $message = 'The provided frequency "{{ string }}" must be a valid parameter for DateInterval::createFromDateString() and must not represent a negative value';
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ class FrequencyValidator extends ConstraintValidator
|
||||
|
||||
$interval = @\DateInterval::createFromDateString($value);
|
||||
if (!$interval) {
|
||||
$this->context->buildViolation($constraint->invalidMessage)
|
||||
$this->context->buildViolation($constraint->message)
|
||||
->setParameter('{{ string }}', $value)
|
||||
->addViolation()
|
||||
;
|
||||
@@ -38,7 +38,7 @@ class FrequencyValidator extends ConstraintValidator
|
||||
$dt->add($interval);
|
||||
|
||||
if ($dt <= $now) {
|
||||
$this->context->buildViolation($constraint->negativeIntervalMessage)
|
||||
$this->context->buildViolation($constraint->message)
|
||||
->setParameter('{{ string }}', $value)
|
||||
->addViolation()
|
||||
;
|
||||
|
||||
Reference in New Issue
Block a user