Commit 3142efe2 authored by Alex Arnaud's avatar Alex Arnaud

dev#17871 - Replace cosmogramme class 'classe_parseur' with DataSource and DataSource parser API

parent 69565830
......@@ -72,6 +72,7 @@ class maj_auto
foreach($lignes as $ligne)
{
$id_bib = $ligne["id_bib"];
$id_prog = $ligne["id_prog"];
$profil = $ligne["profil"];
$type_fichier=fetchOne("select type_fichier from profil_donnees where id_profil=$profil");
$type_operation = $ligne["type_operation"];
......@@ -100,8 +101,8 @@ class maj_auto
{
$log->ecrire('<span class="vert">transfert vers '.$ficNew .'</span></td>');
$date=dateDuJour(0);
$sql->execute("insert into integrations(id_bib,type_operation,profil,type_doc,date_transfert,fichier,traite)
Values($id_bib,$type_operation,$profil,'$type_doc','$date','$ficNew','non')");
$sql->execute("insert into integrations(id_bib,type_operation,profil,type_doc,date_transfert,fichier,traite,int_maj_auto_id)
Values($id_bib,$type_operation,$profil,'$type_doc','$date','$ficNew','non','$id_prog')");
setVariable("ID_upload",$id_upload);
}
else
......
......@@ -218,18 +218,17 @@ while ($ligne = $sql->fetchNext($resultat)) {
$chrono100notices->start();
$chrono_fichier->start();
}
if (false == $parseur->open($integration_path . $fichier, $format, $pointeur_reprise))
{
$integration = Class_Cosmogramme_Integration::find($id);
$data_source = $integration->newDataSource();
if (false == $data_source->open()) {
incrementeVariable("traitement_erreurs");
$log->ecrire('<span class="rouge">Impossible d\'ouvrir le fichier : ' . $fichier . '</span><br>');
continue;
}
// Parser les enregs
while (true)
{
if (!$mode_cron and $chrono->tempsPasse() > $timeout) sauveContexte();
$ret = $parseur->nextEnreg();
$ret = $data_source->nextRecord();
if ($ret["statut"] == "erreur")
{
incrementeVariable("traitement_erreurs");
......
<?php
$adapter = Zend_Db_Table::getDefaultAdapter();
$adapter->query('alter table integrations add int_maj_auto_id INT(11) not null');
?>
......@@ -28,12 +28,18 @@ class Class_Cosmogramme_Integration extends Storm_Model_Abstract {
const TYPE_OPERATION_TOTAL = 2;
const TYPE_OPERATION_PERGAME_HEADER_DELETE = 3;
const PENDING = 'non';
protected $_table_name = 'integrations';
protected $_belongs_to = [
'bib' => ['model' => 'Class_IntBib',
'role' => 'int_bib',
'referenced_in' => 'id_bib'],
'int_maj_auto' => ['model' => 'Class_IntMajAuto',
'role' => 'integration',
'referenced_in' => 'int_maj_auto_id'],
'profil_donnees' => ['model' => 'Class_Cosmogramme_ProfilDonnees',
'role' => 'integration',
'referenced_in' => 'profil']];
......@@ -101,5 +107,15 @@ class Class_Cosmogramme_Integration extends Storm_Model_Abstract {
public function shouldClean() {
return $this->isTotal() && !$this->getPointeurReprise();
}
public function newDataSource() {
return $this->getIntMajAuto()->newDataSource($this);
}
public function newParser() {
return $this->getProfilDonnees()->newParser();
}
}
?>
\ No newline at end of file
......@@ -21,10 +21,25 @@
class Class_Cosmogramme_ProfilDonnees extends Storm_Model_Abstract {
const FILE_TYPE_RECORDS = 0;
const FILE_TYPE_USERS = 1;
const FILE_TYPE_LOANS = 2;
const FILE_TYPE_HOLDS = 3;
const
FILE_TYPE_RECORDS = 0,
FILE_TYPE_USERS = 1,
FILE_TYPE_LOANS = 2,
FILE_TYPE_HOLDS = 3,
FORMAT_UNIMARC = 0,
FORMAT_TABBED_ASCII = 1,
FORMAT_SEMI_COLON_ASCII = 2,
FORMAT_PIPED_ASCII = 3,
FORMAT_XML = 4,
FORMAT_CSV = 5,
FORMAT_MARC21 = 6;
protected $_parser_classes = [
self::FILE_TYPE_RECORDS => [
self::FORMAT_UNIMARC => 'Class_Integration_DataSource_Parser_Record_Iso2709',
self::FORMAT_MARC21 => 'Class_Integration_DataSource_Parser_Record_Iso2709'
]
];
protected $_table_name = 'profil_donnees';
protected $_table_primary = 'id_profil';
......@@ -32,5 +47,16 @@ class Class_Cosmogramme_ProfilDonnees extends Storm_Model_Abstract {
public function isHolds() {
return self::FILE_TYPE_HOLDS == $this->getTypeFichier();
}
public function newParser() {
$ft = $this->getTypeFichier();
$format = $this->getFormat();
if ($parser_class = $this->_parser_classes[$ft][$format])
return new $parser_class();
return null;
}
}
?>
\ No newline at end of file
......@@ -30,6 +30,17 @@ class Class_IntMajAuto extends Storm_Model_Abstract {
protected $_belongs_to = ['int_bib' => ['model' => 'Class_IntBib',
'referenced_in' => 'id_bib']];
public function newDataSource($integration) {
$source_class = $this->isOAI() ? 'Class_Integration_DataSource_OAI' : 'Class_Integration_DataSource_File';
return new $source_class($integration);
}
protected function isOAI() {
return false;
}
}
?>
\ No newline at end of file
<?php
/**
* Copyright (c) 2012, Agence Française Informatique (AFI). All rights reserved.
*
* BOKEH is free software; you can redistribute it and/or modify
* it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE as published by
* the Free Software Foundation.
*
* There are special exceptions to the terms and conditions of the AGPL as it
* is applied to this software (see README file).
*
* BOKEH is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU AFFERO GENERAL PUBLIC LICENSE
* along with BOKEH; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
abstract class Class_Integration_DataSource_Abstract {
protected $_uri;
protected $_parser;
public function __construct($integration) {
$this->_parser = $integration->newParser();
}
public function getParser() {
return $this->_parser;
}
abstract function open();
abstract function nextRecord();
public function setUri($uri) {
$this->_uri = $uri;
}
}
<?php
/**
* Copyright (c) 2012, Agence Française Informatique (AFI). All rights reserved.
*
* BOKEH is free software; you can redistribute it and/or modify
* it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE as published by
* the Free Software Foundation.
*
* There are special exceptions to the terms and conditions of the AGPL as it
* is applied to this software (see README file).
*
* BOKEH is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU AFFERO GENERAL PUBLIC LICENSE
* along with BOKEH; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
class Class_Integration_DataSource_File extends Class_Integration_DataSource_Abstract {
use Trait_StaticFileSystem;
protected $_file;
protected $hFic;
protected $pointeur_reprise;
protected $last_error="";
public function __construct($integration) {
parent::__construct($integration);
$this->_file = $integration->getFichier();
}
public function open() {
$fileSystem = $this->getFileSystem();
$this->hFic = @$fileSystem->fopen($this->_file, 'rb');
if(!$this->hFic) {
$this->last_error = 'Impossible d\'ouvrit le fichier :' . $fic;
return false;
}
$this->getParser()->setWorkingSource($this->hFic);
}
public function nextRecord() {
if (!$data = $this->getParser()->nextRecord())
return ['statut' => 'eof'];
return ['data' => $data, 'statut' => 'ok'];
}
public function close() {
if($this->hFic)
$this->getFileSystem()->fclose($this->hFic);
unset($this->hFic);
}
public function getLastError() {
return $this->last_error;
}
}
<?php
/**
* Copyright (c) 2012, Agence Française Informatique (AFI). All rights reserved.
*
* BOKEH is free software; you can redistribute it and/or modify
* it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE as published by
* the Free Software Foundation.
*
* There are special exceptions to the terms and conditions of the AGPL as it
* is applied to this software (see README file).
*
* BOKEH is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU AFFERO GENERAL PUBLIC LICENSE
* along with BOKEH; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
class Class_Integration_DataSource_OAI extends Class_Integration_DataSource_Abstract {
protected $_uri;
protected $_response;
protected $_records;
protected $_current_record;
protected $_http_client;
const ListRecordsVerb = 'ListRecords';
public function __construct($integration, $profile) {
$this->setUri($integration->getFichier());
$this->_http_client = new Class_WebService_SimpleWebClient();
$this->xml_parser = Class_WebService_XMLParser::newInstance();
$this->xml_parser->keepNameSpace();
$this->xml_parser->setElementHandler($this);
}
public function open() {
$this->_response = $this->httpGet($url);
$this->parseResponse();
}
public function nextRecord() {
}
public function parseResponse() {
return $this->parse($this->_response);
}
public function parse($xml) {
$this->xml_parser->parse($xml);
return $this;
}
public function getSets() {
return $this->oaiAsks(self::ListSetsVerb)
->parseResponse()
->getData();
}
public function startRecord() {
}
public function startSet($attributes) {
$this->_current_set = [];
}
public function endSetSpec($data) {
$this->_current_set['code'] = $data;
}
public function endSetName($data) {
$this->_current_set['label'] = $data;
}
public function endSet() {
$this->_data[] = $this->_current_set;
}
public function setHttpClient($client) {
$this->_http_client = $client;
return $this;
}
public static function getHttpClient() {
if ($this->_http_client == null || !isset($this->_http_client)) {
$this->_http_client = new Class_WebService_SimpleWebClient();
}
return $this->_http_client;
}
public function httpGet($url) {
return $this->getHttpClient()->open_url($url);
}
public function setUri($uri) {
$this->_uri = $uri;
return $this;
}
}
<?php
/**
* Copyright (c) 2012, Agence Française Informatique (AFI). All rights reserved.
*
* BOKEH is free software; you can redistribute it and/or modify
* it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE as published by
* the Free Software Foundation.
*
* There are special exceptions to the terms and conditions of the AGPL as it
* is applied to this software (see README file).
*
* BOKEH is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU AFFERO GENERAL PUBLIC LICENSE
* along with BOKEH; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
abstract class Class_Integration_DataSource_Parser_Abstract {
protected $_working_source;
abstract function nextRecord();
public function setWorkingSource($data_source) {
$this->_working_source = $data_source;
}
}
<?php
/**
* Copyright (c) 2012, Agence Française Informatique (AFI). All rights reserved.
*
* BOKEH is free software; you can redistribute it and/or modify
* it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE as published by
* the Free Software Foundation.
*
* There are special exceptions to the terms and conditions of the AGPL as it
* is applied to this software (see README file).
*
* BOKEH is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU AFFERO GENERAL PUBLIC LICENSE
* along with BOKEH; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
class Class_Integration_DataSource_Parser_Record_Iso2709 extends Class_Integration_DataSource_Parser_Abstract {
const CHUNK_LENGTH = 4096;
protected $_buffer;
public function nextRecord() {
$data = $this->_buffer;
$fileSystem = $this->getFileSystem();
while(true) {
$sep = chr(30) . chr(29);
$pos = strpos($data, $sep);
if(false !== $pos) {
$this->_buffer = substr($data, ($pos+2));
$data = substr($data, 0, ($pos+2));
break;
}
$enreg = $fileSystem->fread($hFic, CHUNK_LENGTH);
if($fileSystem->feof($hFic)) {
if($enreg == false) {
return '';
}
}
$data .= $enreg;
}
return $data;
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment