dev#100949 : import events from openagenda (JSON Format)

FEATURES/100949

+        '100949' =>
+            ['Label' => $this->_('Intégration d'Evénements publiés dans un Open-Agenda (JSON)'),
+             'Desc' => $this->_('Les Calendriers Externes au format OpenAgenda peuvent être importés.'),
+             'Image' => '',
+             'Video' => 'https://youtube.com/watch?v=5leZZihcMKk',
+             'Category' => $this->_('Import Calendriers'),
+             'Right' => function($feature_description, $user) {return true;},
+             'Wiki' => 'http://wiki.bokeh-library-portal.org/index.php?title=Agendas_Externes',
+             'Test' => 'http://git.afi-sa.fr/afi/opacce/merge_requests/3329',
+             'Date' => '2019-11-26'],
\ No newline at end of file

VERSIONS_WIP/100949

+ - ticket #100949 : Agenda Externe : Intégration d'événements publiés sur Open-Agenda (format JSON)
\ No newline at end of file

application/modules/admin/controllers/ExternalAgendasController.php

@@ -32,9 +32,12 @@ class Admin_ExternalAgendasController extends ZendAfi_Controller_Action {
       return $this->_redirectToIndex();
 
     $this->view->titre = $this->_('Moissonnage des évènements de l\'agenda "%s"', $agenda->getLibelle());
-    $results = $agenda->import();
-    $this->view->new_events = $results['new'];
-    $this->view->updated_events = $results['update'];
+
+    $agenda->import(function($created, $updated)
+                    {
+                      $this->view->new_events = $created;
+                      $this->view->updated_events = $updated;
+                    });
   }
 
 

cosmogramme/sql/patch/patch_381.php

+<?php
+$adapter = Zend_Db_Table_Abstract::getDefaultAdapter();
+try {
+  $adapter->query("alter table external_agenda
+                       add column `provider` text not null default ''" );
+} catch (Exception $e) {}
+try {
+  $adapter->query("alter table external_agenda
+                       add column `delete_orphan_events` tinyint(1) not null default 0" );
+} catch (Exception $e) {}

library/Class/Batch/ExternalAgenda.php

@@ -31,4 +31,3 @@ class Class_Batch_ExternalAgenda extends Class_Batch_Abstract {
     Class_ExternalAgenda::harvest($this->getLogger());
   }
 }
-?>
\ No newline at end of file

library/Class/ExternalAgenda.php

@@ -21,22 +21,32 @@
 
 
 class Class_ExternalAgendaLoader extends Storm_Model_Loader {
+  use Trait_Translator;
 
   public function harvest($logger) {
-    foreach (Class_ExternalAgenda::findAllBy(['autoharvest' => 1]) as $agenda) {
-      $results = $agenda->import();
-      $logger->log($agenda->getLabel().":\n");
-      $logger->log($agenda->_("Nombre d\'événements créés : %s\n",count($results['new'])));
-      Class_AdminVar::get('AGENDA_KEEP_LOCAL_CONTENT')
-        ? $logger->log($agenda->_("Nombre d\'événements non mis à jour : %s\n",count($results['update'])))
-        : $logger->log($agenda->_("Nombre d\'événements mis à jour : %s\n",count($results['update'])));
-    }
+    foreach (Class_ExternalAgenda::findAllBy(['autoharvest' => 1]) as $agenda)
+      $agenda
+        ->import(function($created, $updated) use($logger, $agenda)
+                 {
+                   $agenda->logImportOn($logger, $created->count(), $updated->count());
+                 });
+  }
+
+
+  public function getAllProviderOptions() {
+    return [ Class_ExternalAgenda::ICALENDAR => $this->_('iCalendar (.ics)'),
+            Class_ExternalAgenda::OPEN_AGENDA => $this->_('OpenAgenda (.json)')];
   }
 }
 
 
+
+
 class Class_ExternalAgenda extends Storm_Model_Abstract {
-  use Trait_Translator;
+  use Trait_Translator, Trait_TimeSource;
+
+  const ICALENDAR   = 1;
+  const OPEN_AGENDA = 2;
 
   protected $_table_name = 'external_agenda';
   protected $_belongs_to = ['category' => ['model' => 'Class_ArticleCategorie',
@@ -47,6 +57,13 @@ class Class_ExternalAgenda extends Storm_Model_Abstract {
 
   protected $_loader_class = 'Class_ExternalAgendaLoader';
 
+  protected $_default_attribute_values =
+    [
+     'provider'=> self::ICALENDAR,
+     'delete_orphan_events' => 0
+    ];
+
+
   public function getLibelle() {
     return $this->getLabel();
   }
@@ -59,17 +76,68 @@ class Class_ExternalAgenda extends Storm_Model_Abstract {
   }
 
 
-  public function import() {
-    $service = new Class_WebService_ICalendar();
+  public function import($after_import=null) {
+    $service = $this->_newProvider();
     $events = $service->import($this);
-    $results['new'] = $events->select('isNew');
-    $results['update'] = $events->reject('isNew');
-    $results['new']->eachDo('save');
+    $events->eachDo([$this, 'deduplicateEvent']);
+
+    $created = $events->select('isNew');
+    $updated = $events->reject('isNew');
+
+    // must be after updated detection
+    $created->eachDo(function($model)
+                     {
+                       $model
+                         ->setDateCreation(date('Y-m-d H:i:s', $this->getCurrentTime()))
+                         ->save();
+                     });
 
     if (!Class_AdminVar::get('AGENDA_KEEP_LOCAL_CONTENT'))
-      $results['update']->eachDo('save');
+      $updated->eachDo(function ($model) { $model->updateDateMaj()->save(); });
+
+    if ($this->getDeleteOrphanEvents())
+      $this->_deleteOrphanEvents($events);
+
+    if ($after_import)
+      $after_import($created, $updated);
+  }
+
+
+  public function logImportOn($logger, $created_count, $updated_count) {
+    $logger->log($this->getLabel().":\n");
+    $logger->log($this->_("Nombre d'événements créés : %s\n", $created_count));
+
+    $message = Class_AdminVar::get('AGENDA_KEEP_LOCAL_CONTENT')
+      ? $this->_("Nombre d'événements non mis à jour : %s\n", $updated_count)
+      : $this->_("Nombre d'événements mis à jour : %s\n", $updated_count);
+
+    $logger->log($message);
+  }
+
+
+  public function deduplicateEvent($event){
+    if ($existing_event = $this->findEventByUID($event->getIdOrigine()))
+      $event->setId($existing_event->getId());
+  }
+
+
+  protected function _newProvider() {
+    $map = [static::OPEN_AGENDA => 'Class_ExternalAgenda_OpenAgenda',
+            static::ICALENDAR => 'Class_ExternalAgenda_ICalendar'];
+
+    return array_key_exists($this->getProvider(), $map)
+      ? new $map[$this->getProvider()]
+      : new Class_ExternalAgenda_Provider();
+  }
+
+
+  protected function _deleteOrphanEvents($updated_events) {
+    $delete_params = ['repository_origine' => $this->getRepositoryKey()];
+
+    if (!$updated_events->isEmpty())
+      $delete_params['ID_ARTICLE not'] = $updated_events->collect('id')->getArrayCopy();
 
-    return $results;
+    Class_Article::deleteBy($delete_params);
   }
 
 

library/Class/WebService/ICalendar.php → library/Class/ExternalAgenda/ICalendar.php

@@ -20,34 +20,25 @@
  */
 
 
-class Class_WebService_ICalendar extends Class_WebService_Abstract {
-  use Trait_Translator;
-
+class Class_ExternalAgenda_ICalendar extends Class_ExternalAgenda_Provider {
   protected
-    $_events,
     $_current_event,
-    $_current_url,
-    $_external_agenda;
+    $_current_url;
 
-  public function import($external_agenda) {
-    $this->_external_agenda = $external_agenda;
-    $this->_events = new Storm_Model_Collection();
-    $this->_current_event = null;
 
+  protected function _import() {
+    $this->_current_event = null;
 
-    $ics_content = $this->httpGet($external_agenda->getUrl());
+    $ics_content = $this->httpGet($this->_external_agenda->getUrl());
     $ics_content = preg_replace('|\n\s|', '', $ics_content); //see RFC2445
     $lines = preg_split('|\r?\n|', $ics_content);
 
     array_map([$this, '_importLine'], $lines);
 
-    return $this->_events;
+    return $this;
   }
 
 
-  public function __call($method, $params) {}
-
-
   protected function _importLine($line) {
     if (!$line)
       return $this;
@@ -59,6 +50,14 @@ class Class_WebService_ICalendar extends Class_WebService_Abstract {
   }
 
 
+  public function __call($name, $args) {
+    if ('on' == substr($name, 0, 2))
+      return;
+
+    throw new RuntimeException('Call to undefined method Class_ExternalAgenda_ICalendar::' . $name);
+  }
+
+
   protected function _importData($key, $value) {
     if (in_array($key, ['BEGIN', 'END']))
       return $this->{'on' . $key . $value}();
@@ -116,9 +115,6 @@ class Class_WebService_ICalendar extends Class_WebService_Abstract {
   protected function onEventUID($value) {
     $value = md5($value);
 
-    if ($existing_event = $this->_external_agenda->findEventByUID($value))
-      $this->_current_event->setId($existing_event->getId());
-
     $this->_current_event->setIdOrigine($value);
     $this->_events->append($this->_current_event);
 

library/Class/ExternalAgenda/OpenAgenda.php

+<?php
+/**
+ * Copyright (c) 2012-2019, Agence Française Informatique (AFI). All rights reserved.
+ *
+ * BOKEH is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE as published by
+ * the Free Software Foundation.
+ *
+ * There are special exceptions to the terms and conditions of the AGPL as it
+ * is applied to this software (see README file).
+ *
+ * BOKEH is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU AFFERO GENERAL PUBLIC LICENSE for more details.
+ *
+ * You should have received a copy of the GNU AFFERO GENERAL PUBLIC LICENSE
+ * along with BOKEH; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+ */
+
+
+class Class_ExternalAgenda_OpenAgenda extends Class_ExternalAgenda_Provider {
+
+
+  public function _import() {
+    $this->_loadPage();
+    return $this;
+  }
+
+
+  protected function _loadPage($offset = 0, $event_count = 0){
+    $content = json_decode($this->httpGet($this->_external_agenda->getUrl().'&offset='. (int) $offset),true);
+
+    if (!$content)
+      return $this;
+
+    array_map([$this, '_processEvent'], $content['events']);
+    $event_count += count($content['events']);
+
+    return ($event_count < $content['total'])
+      ? $this->_loadPage( $offset + 1, $event_count)
+      : $this;
+  }
+
+
+  protected function _processEvent($datas){
+    $event = new Class_ExternalAgenda_OpenAgenda_Event($datas);
+
+    foreach($event->get('timings') as $timing) {
+      $article = $this->_buildArticleForTiming($event, $timing);
+      $this->_events->append($article);
+    }
+
+    return $this;
+  }
+
+
+  protected function _buildArticleForTiming($event, $timing) {
+    return $this->_external_agenda
+      ->newEvent()
+      ->setTitre($event->getString('title'))
+      ->setContenu($event->getImageTagWithCredits().$event->getHtml().$event->getInfosTag())
+      ->setIdOrigine($event->get('uid') . '_' . base64_encode($timing['start']))
+      ->setDescription($event->getImageTag().'<p>'.$event->getString('description').'</p>')
+      ->setTags(implode(';', $event->getKeywords()))
+      ->setLieu($event->getLocation())
+      ->setEventsDebut(date('Y-m-d H:i', strtotime($timing['start'])))
+      ->setEventsFin(date('Y-m-d H:i', strtotime($timing['end'])));
+  }
+}
+
+
+
+
+class Class_ExternalAgenda_OpenAgenda_Event {
+  use Trait_Translator;
+
+  protected
+    $_event;
+
+  public function __construct($event){
+    $this->_event = $event;
+  }
+
+  public function getImageTagWithCredits(){
+    $imgsrc = $this->getImageTag();
+    return ($credits = $this->getString('imageCredits'))
+      ? sprintf('<figure>%s<figcaption>%s</figcaption></figure>',
+                $imgsrc,
+                $this->_('Credits : %s', $credits))
+      : $imgsrc;
+  }
+
+
+  public function getImageTag(){
+    return ($src = $this->_event['image'])
+      ? sprintf('<img src="%s" alt=""/>',$src)
+      : '';
+  }
+
+
+  public function getInfosTag() {
+    $infos = '';
+
+    $infos .= $this->_prepareConditionsString();
+    $infos .= $this->_prepareAgeString();
+
+    if (!$infos)
+      return '';
+
+    return '<p>' . $this->_('Infos pratiques :') . '</p><dl>' . $infos . '</dl>';
+  }
+
+  protected function _addInfoElement($label, $description){
+    return '<dt>' . $label . '</dt>'
+        . '<dd>' . $description . '</dd>';
+  }
+
+
+  public function getKeywords(){
+    if (! $keywords = $this->getArray('keywords'))
+      return [];
+
+    return isset( $keywords['fr'] )
+      ? $keywords['fr']
+      : [];
+  }
+
+  protected function _prepareConditionsString() {
+    return ($conditions = $this->getString('conditions'))
+      ? $this->_addInfoElement(
+                               $this->_('Conditions'),
+                               $conditions)
+      : "";
+  }
+
+
+  protected function _prepareAgeString() {
+    return ($this->_event['age'])
+      ? $this->_addInfoElement(
+                               $this->_('Âge'),
+                               $this->_('de %s à %s ans',
+                                        $this->_event['age']['min'],
+                                        $this->_event['age']['max']))
+      : "";
+  }
+
+
+
+
+  public function getString($name){
+    if (!(isset($this->_event[$name]) && $this->_event[$name]))
+      return '';
+
+    if (is_string($this->_event[$name]))
+      return $this->_event[$name];
+
+    return is_array($this->_event[$name])
+      ? reset($this->_event[$name])
+      : '';
+  }
+
+
+  public function getArray($name){
+    return ($data = $this->get($name)) && is_array($data)
+      ? $data
+      : [];
+  }
+
+
+  public function get($name) {
+    return $this->_event[$name];
+  }
+
+
+  public function getHtml() {
+    return ($description_html = $this->getString('html'))
+      ? $description_html
+      : $this->getString('description');
+  }
+
+
+  public function getLocation(){
+    if ($lieu = Class_Lieu::findFirstBy(['latitude'  => $this->get('latitude'),
+                                         'longitude' => $this->get('longitude')]))
+      return $lieu;
+
+    $lieu = new Class_Lieu();
+    $lieu
+      ->setLibelle($this->getString('locationName'))
+      ->setLatitude($this->get('latitude'))
+      ->setLongitude($this->get('longitude'))
+      ->setAdresse(implode(',', explode(',', $this->getString('address'), -1)))
+      ->setCodePostal($this->getString('postalCode'))
+      ->setVille($this->getString('city'));
+
+    $location = $this->getArray('location');
+    $lieu
+      ->setTelephone(isset($location['phone']) ? (string)$location['phone'] : '')
+      ->setMail(isset($location['email']) ? (string)$location['email'] : '')
+      ->setUrl(isset($location['website']) ? (string)$location['website'] : '')
+      ->save();
+    return $lieu;
+  }
+}

library/Class/ExternalAgenda/Provider.php

+<?php
+/**
+ * Copyright (c) 2012-2019, Agence Française Informatique (AFI). All rights reserved.
+ *
+ * BOKEH is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE as published by
+ * the Free Software Foundation.
+ *
+ * There are special exceptions to the terms and conditions of the AGPL as it
+ * is applied to this software (see README file).
+ *
+ * BOKEH is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU AFFERO GENERAL PUBLIC LICENSE for more details.
+ *
+ * You should have received a copy of the GNU AFFERO GENERAL PUBLIC LICENSE
+ * along with BOKEH; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+ */
+
+
+class Class_ExternalAgenda_Provider extends Class_WebService_Abstract {
+  use Trait_Translator;
+
+  protected
+    $_events,
+    $_external_agenda;
+
+
+  public function import($external_agenda) {
+    $this->_events = new Storm_Model_Collection();
+    $this->_external_agenda = $external_agenda;
+
+    $this->_import();
+
+    return $this->_events;
+  }
+
+
+  protected function _import() {
+    return $this;
+  }
+}

library/ZendAfi/Form/Admin/ExternalAgenda.php

@@ -42,6 +42,10 @@ class ZendAfi_Form_Admin_ExternalAgenda extends ZendAfi_Form {
                    'autoharvest',
                    ['label' => $this->_('Moissonnage automatique'),
                     ])
+      ->addElement('checkbox',
+                   'delete_orphan_events',
+                   ['label' => $this->_('Supprimer localement les événements supprimés dans le calendrier source'),
+                    ])
 
       ->addElement('comboCategories',
                    'cat_id',
@@ -54,9 +58,22 @@ class ZendAfi_Form_Admin_ExternalAgenda extends ZendAfi_Form {
       ->addElement('select',
                    'id_lieu',
                    ['label' => $this->_('Lieu'),
-                    'multiOptions' => ['0' => $this->_('Aucun')] + Class_Lieu::getAllLibelles()]);
+                    'multiOptions' => ['0' => $this->_('Aucun')] + Class_Lieu::getAllLibelles()])
+      ->addElement('select',
+                   'provider',
+                   ['label' => $this->_('Source'),
+                    'multiOptions' => Class_ExternalAgenda::getAllProviderOptions()]);
 
-    $elements = ['label', 'url','autoharvest', 'cat_id', 'id_lieu'];
+    $elements =
+      [
+       'label',
+       'provider',
+       'url',
+       'autoharvest',
+       'delete_orphan_events',
+       'cat_id',
+       'id_lieu'
+      ];
 
     if (Class_AdminVar::isWorkFlowEnabled()) {
       $this->addElement('radio', 'status',
@@ -68,5 +85,7 @@ class ZendAfi_Form_Admin_ExternalAgenda extends ZendAfi_Form {
     }
 
     $this->addDisplayGroup($elements, 'agenda', ['legend' => $this->_('Agenda')]);
+    Class_ScriptLoader::getInstance()
+      ->addJqueryReady('formSelectToggleVisibilityForElement( "#provider", $("select[name=\'id_lieu\']").closest("tr"), ["1"]);');
   }
 }

tests/application/modules/opac/controllers/RecordCustomLinksTest.php

@@ -118,6 +118,8 @@ class RecordCustomLinksRechercheControllerWithBrazilTest extends RecordCustomLin
   public function setUp() {
     parent::setUp();
 
+    Class_AdminVar::set('FEATURES_TRACKING_ENABLE', '0');
+
     $notice = $this->fixture('Class_Notice', ['id' => '888',
                                               'type_doc' => Class_CodifTypeDoc::SONORE,
                                               'unimarc' => file_get_contents(__DIR__ . '/../../../../fixtures/unimarc_brazil.txt')]);
@@ -128,9 +130,17 @@ class RecordCustomLinksRechercheControllerWithBrazilTest extends RecordCustomLin
                                                             'liste_codes' => "TAN98"],
                                        'viewnotice3' => ['links_zones' => '856-u-a']]]);
 
+
+    $this->mock_sql = $this->mock()
+                           ->whenCalled('fetchAll')
+                           ->answers([ [888, ''] ]);
+    Zend_Registry::set('sql', $this->mock_sql);
+
     Storm_Test_ObjectWrapper::onLoaderOfModel('Class_Notice')
-      ->whenCalled('findAllBy')
-      ->answers([Class_Notice::find(888)]);
+      ->whenCalled('findAllByIds')
+      ->with([888], 10, null)
+      ->answers([Class_Notice::find(888)])
+      ->beStrict();
 
     $this->dispatch('/opac/recherche/simple/expressionRecherche/brazil');
   }

tests/db/UpgradeDBTest.php

@@ -2967,3 +2967,26 @@ class UpgradeDB_380_Test extends UpgradeDBTestCase {
     $this->assertIndex('hold_pnb', 'subscriber_id');
   }
 }
+
+
+
+
+class UpgradeDB_381_Test extends UpgradeDBTestCase {
+  public function prepare() {
+    $this
+      ->silentQuery('ALTER TABLE external_agenda DROP COLUMN provider')
+      ->silentQuery('ALTER TABLE external_agenda DROP COLUMN delete_orphan_events');
+  }
+
+
+  /** @test */
+  public function tableExternalAgendasShouldHaveColumnProviderText() {
+    $this->assertFieldType('external_agenda', 'provider', 'text');
+  }
+
+
+  /** @test */
+  public function tableExternalAgendasShouldHaveColumnDeleteOrphanEvents() {
+    $this->assertFieldType('external_agenda', 'delete_orphan_events', 'tinyint(1)');
+  }
+}