From fd7787bf18bb4bca63e71b823c93e2e36fc2e34b Mon Sep 17 00:00:00 2001
From: Ludwig Krispenz <lkrispen@redhat.com>
Date: Wed, 31 Jan 2018 09:22:10 +0100
Subject: [PATCH 3/3] move cldb - export changelog with backend

---
 ldap/admin/src/scripts/db2ldif.in             |  4 +-
 ldap/admin/src/scripts/ldif2db.in             |  4 +-
 ldap/servers/plugins/replication/cl5_api.c    | 59 +++++++++++++++++++++++++++
 ldap/servers/plugins/replication/cl5_api.h    |  3 ++
 ldap/servers/plugins/replication/repl5_init.c |  2 +
 ldap/servers/slapd/back-ldbm/ldif2ldbm.c      |  8 ++++
 ldap/servers/slapd/main.c                     | 20 ++++++++-
 ldap/servers/slapd/pblock.c                   | 35 ++++++++++++++++
 ldap/servers/slapd/pblock_v3.h                |  1 +
 ldap/servers/slapd/plugin.c                   |  4 ++
 ldap/servers/slapd/slap.h                     |  4 ++
 ldap/servers/slapd/slapi-plugin.h             |  3 ++
 12 files changed, 144 insertions(+), 3 deletions(-)

diff --git a/ldap/admin/src/scripts/db2ldif.in b/ldap/admin/src/scripts/db2ldif.in
index 95d2754..6e3af53 100755
--- a/ldap/admin/src/scripts/db2ldif.in
+++ b/ldap/admin/src/scripts/db2ldif.in
@@ -25,6 +25,7 @@ usage()
     echo "        -x                - Suffix to exclude"
     echo "        -a outputfile     - Name of the exported LDIF file"
     echo "        -r                - Include replication data"
+    echo "        -R                - Include changelog data"
     echo "        -E                - Decrypt attributes"
     echo "        -u                - Do not export the nsUniqueId attribute"
     echo "        -U                - Do not wrap long lines"
@@ -99,7 +100,7 @@ then
     exit 1
 fi
 
-while getopts "hZ:vd:D:ENa:rs:x:CSut:n:UmMo1qc:" flag
+while getopts "hZ:vd:D:ENa:rs:x:CSut:n:UmMo1qRc:" flag
 do
     case $flag in
         h) usage
@@ -125,6 +126,7 @@ do
         M) args=$args" -M";;
         1) args=$args" -1";;
         q) args=$args" -q";;        
+        R) args=$args" -R";;        
         c) cwd=$OPTARG;;
         ?) usage
            exit 1;;
diff --git a/ldap/admin/src/scripts/ldif2db.in b/ldap/admin/src/scripts/ldif2db.in
index 20c7d46..9097d16 100755
--- a/ldap/admin/src/scripts/ldif2db.in
+++ b/ldap/admin/src/scripts/ldif2db.in
@@ -30,6 +30,7 @@ usage()
     echo "        -G name           - Namespace id for name based uniqueid (-g deterministic)"
     echo "        -O                - Do not index the attributes"
     echo "        -E                - Encrypt attributes"
+    echo "        -R                - Import changelog data"
     echo "        -q                - Quiet mode - suppresses output"
     echo "        -v                - Display version"
     echo "        -h                - Display usage" 
@@ -53,7 +54,7 @@ handleopts()
     return 0
 }
 
-while getopts "Z:vhd:i:g:G:n:s:x:NOCc:St:D:Eq" flag
+while getopts "Z:vhd:i:g:G:n:s:x:NOCc:St:D:ERq" flag
 do
     case $flag in
         h) usage
@@ -70,6 +71,7 @@ do
         t) args=$args" -t \"$OPTARG\"";;
         D) args=$args" -D \"$OPTARG\"";;
         E) args=$args" -E";;
+        R) args=$args" -R";;
         v) args=$args" -v";;
         N) args=$args" -N";;
         C) args=$args" -C";;
diff --git a/ldap/servers/plugins/replication/cl5_api.c b/ldap/servers/plugins/replication/cl5_api.c
index 7320316..4c5a4e5 100644
--- a/ldap/servers/plugins/replication/cl5_api.c
+++ b/ldap/servers/plugins/replication/cl5_api.c
@@ -4972,6 +4972,65 @@ bail:
     return rc;
 }
 
+static char *
+_cl5LdifFileName(char *instance_ldif)
+{
+    char *cl_ldif = NULL;
+    char *p =strstr(instance_ldif, ".ldif");
+
+    if (p) {
+        *p = '\0';
+        cl_ldif = slapi_ch_smprintf("%s_cl.ldif", instance_ldif);
+    } else {
+        cl_ldif = slapi_ch_smprintf("%s_cl", instance_ldif);
+    }
+
+    return cl_ldif;
+
+}
+
+int
+cl5Import(Slapi_PBlock *pb)
+{
+    slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
+                          "cl5Export - Importing changelog\n");
+    return 0;
+}
+
+int
+cl5Export(Slapi_PBlock *pb)
+{
+    char *instance_name;
+    char *instance_ldif;
+    char *instance_cl_ldif;
+    Slapi_Backend *be;
+    Object *repl_obj;
+    Replica *replica = NULL;
+    int rc;
+
+
+
+    slapi_pblock_get(pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name);
+    slapi_pblock_get(pb, SLAPI_DB2LDIF_FILE, &instance_ldif);
+    slapi_pblock_get(pb, SLAPI_BACKEND, &be);
+    if (repl_obj = replica_get_replica_from_dn(slapi_be_getsuffix(be, 0))) {
+       replica = (Replica *)object_get_data(repl_obj);
+    }
+    if (replica == NULL) {
+        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
+                              "cl5Export - No replica defined for instance %s\n", instance_name);
+        return 0;
+    }
+
+    instance_cl_ldif = _cl5LdifFileName(instance_ldif);
+    slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
+                  "cl5Export - Exporting changelog for instance %s to file %s\n",
+                  instance_name, instance_cl_ldif);
+    rc = cl5ExportLDIF(instance_cl_ldif, replica);
+
+    return rc;
+}
+
 /*
  *  Clean the in memory RUV, at shutdown we will write the update to the db
  */
diff --git a/ldap/servers/plugins/replication/cl5_api.h b/ldap/servers/plugins/replication/cl5_api.h
index ec22571..0a58098 100644
--- a/ldap/servers/plugins/replication/cl5_api.h
+++ b/ldap/servers/plugins/replication/cl5_api.h
@@ -420,6 +420,9 @@ int cl5WriteRUV(void);
    Parameters:    none
    Return:        TRUE
 */
+int cl5Export(Slapi_PBlock *pb);
+int cl5Import(Slapi_PBlock *pb);
+
 int cl5DeleteRUV(void);
 void cl5CleanRUV(ReplicaId rid, Replica *replica);
 void cl5NotifyCleanup(int rid);
diff --git a/ldap/servers/plugins/replication/repl5_init.c b/ldap/servers/plugins/replication/repl5_init.c
index 6afd0fe..6b118ac 100644
--- a/ldap/servers/plugins/replication/repl5_init.c
+++ b/ldap/servers/plugins/replication/repl5_init.c
@@ -407,6 +407,8 @@ multimaster_betxn_bepostop_init(Slapi_PBlock *pb)
 
     if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, SLAPI_PLUGIN_VERSION_01) ||
         slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, (void *)&multimasterbepostopdesc) ||
+        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_EXPORT_FN, (void *)cl5Export) ||
+        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_IMPORT_FN, (void *)cl5Import) ||
         slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_BACKUP_FN, (void *)cl5DeleteRUV)) {
         slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name, "multimaster_betxn_bepostop_init - Failed\n");
         rc = -1;
diff --git a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c
index 16b87ee..261bb03 100644
--- a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c
+++ b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c
@@ -1115,6 +1115,7 @@ ldbm_back_ldbm2ldif(Slapi_PBlock *pb)
     int decrypt = 0;
     int32_t dump_replica = 0;
     int dump_uniqueid = 1;
+    int dump_changelog = 0;
     int fd = STDOUT_FILENO;
     IDList *idl = NULL; /* optimization for -s include lists */
     int cnt = 0, lastcnt = 0;
@@ -1231,6 +1232,7 @@ ldbm_back_ldbm2ldif(Slapi_PBlock *pb)
     slapi_pblock_get(pb, SLAPI_DB2LDIF_FILE, &fname);
     slapi_pblock_get(pb, SLAPI_DB2LDIF_PRINTKEY, &printkey);
     slapi_pblock_get(pb, SLAPI_DB2LDIF_DUMP_UNIQUEID, &dump_uniqueid);
+    slapi_pblock_get(pb, SLAPI_LDIF_CHANGELOG, &dump_changelog);
 
     /* tsk, overloading printkey.  shame on me. */
     ok_index = !(printkey & EXPORT_ID2ENTRY_ONLY);
@@ -1629,6 +1631,12 @@ ldbm_back_ldbm2ldif(Slapi_PBlock *pb)
                       "export %s: Processed %d entries (100%%).\n",
                       inst->inst_name, cnt);
     }
+    if (run_from_cmdline && dump_changelog) {
+        return_value = plugin_call_plugins(pb, SLAPI_PLUGIN_BE_POST_EXPORT_FN);
+        slapi_log_err(SLAPI_LOG_INFO, "ldbm_back_ldbm2ldif",
+                      "export changelog for %s.\n", inst->inst_name);
+    }
+
 bye:
     if (idl) {
         idl_free(&idl);
diff --git a/ldap/servers/slapd/main.c b/ldap/servers/slapd/main.c
index 521d314..99042be 100644
--- a/ldap/servers/slapd/main.c
+++ b/ldap/servers/slapd/main.c
@@ -85,6 +85,7 @@ struct main_config
     int ldif2db_noattrindexes;
     char **db2index_attrs;
     int ldif_printkey;
+    int ldif_include_changelog;
     char *archive_name;
     int db2ldif_dump_replica;
     int db2ldif_dump_uniqueid;
@@ -642,6 +643,7 @@ main(int argc, char **argv)
     /* Set a number of defaults */
     mcfg.slapd_exemode = SLAPD_EXEMODE_UNKNOWN;
     mcfg.ldif_printkey = EXPORT_PRINTKEY | EXPORT_APPENDMODE;
+    mcfg.ldif_include_changelog = 0;
     mcfg.db2ldif_dump_uniqueid = 1;
     mcfg.ldif2db_generate_uniqueid = SLAPI_UNIQUEID_GENERATE_TIME_BASED;
     mcfg.ldif2db_removedupvals = 1;
@@ -1276,7 +1278,7 @@ process_command_line(int argc, char **argv, struct main_config *mcfg)
      *
      */
 
-    char *opts_db2ldif = "vd:D:ENa:rs:x:CSut:n:UmMo1q";
+    char *opts_db2ldif = "vd:D:ENa:rs:x:CSut:n:UmMo1qR";
     struct opt_ext long_options_db2ldif[] = {
         {"version", ArgNone, 'v'},
         {"debug", ArgRequired, 'd'},
@@ -1296,6 +1298,7 @@ process_command_line(int argc, char **argv, struct main_config *mcfg)
         {"multipleOutputFile", ArgNone, 'M'},
         {"noVersionNum", ArgNone, '1'},
         {"quiet", ArgNone, 'q'},
+        {"includechangelog", ArgNone, 'R'},
         {0, 0, 0}};
 
     char *opts_ldif2db = "vd:i:g:G:n:s:x:NOCc:St:D:Eq";
@@ -1692,6 +1695,20 @@ process_command_line(int argc, char **argv, struct main_config *mcfg)
 
             break;
 
+        case 'R': /* db2ldif  and ldif2db only */
+            if (mcfg->slapd_exemode != SLAPD_EXEMODE_DB2LDIF &&
+                mcfg->slapd_exemode != SLAPD_EXEMODE_LDIF2DB) {
+                usage(mcfg->myname, mcfg->extraname, mcfg->slapd_exemode);
+                exit(1);
+            }
+
+            /*
+             * import/export should handle changelog.
+             */
+            mcfg->ldif_include_changelog = 1;
+
+            break;
+
         case 'C':
             if (mcfg->slapd_exemode == SLAPD_EXEMODE_LDIF2DB) {
                 /* used to mean "Cool new import" (which is now
@@ -2266,6 +2283,7 @@ slapd_exemode_db2ldif(int argc, char **argv, struct main_config *mcfg)
         slapi_pblock_set(pb, SLAPI_BACKEND_INSTANCE_NAME, *instp);
         slapi_pblock_set_ldif_dump_replica(pb, mcfg->db2ldif_dump_replica);
         slapi_pblock_set(pb, SLAPI_DB2LDIF_DUMP_UNIQUEID, &(mcfg->db2ldif_dump_uniqueid));
+        slapi_pblock_set(pb, SLAPI_LDIF_CHANGELOG, &(mcfg->ldif_include_changelog));
         int32_t task_flags = SLAPI_TASK_RUNNING_FROM_COMMANDLINE;
         slapi_pblock_set(pb, SLAPI_TASK_FLAGS, &task_flags);
         int32_t is_running = 0;
diff --git a/ldap/servers/slapd/pblock.c b/ldap/servers/slapd/pblock.c
index 4514c3c..6482a61 100644
--- a/ldap/servers/slapd/pblock.c
+++ b/ldap/servers/slapd/pblock.c
@@ -1276,6 +1276,18 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value)
         }
         (*(IFP *)value) = pblock->pb_plugin->plg_bepostbackup;
         break;
+    case SLAPI_PLUGIN_BE_POST_EXPORT_FN:
+        if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) {
+            return (-1);
+        }
+        (*(IFP *)value) = pblock->pb_plugin->plg_bepostexport;
+        break;
+    case SLAPI_PLUGIN_BE_POST_IMPORT_FN:
+        if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) {
+            return (-1);
+        }
+        (*(IFP *)value) = pblock->pb_plugin->plg_bepostimport;
+        break;
 
     /* internal preoperation plugin */
     case SLAPI_PLUGIN_INTERNAL_PRE_MODIFY_FN:
@@ -2090,6 +2102,13 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value)
             (*(int *)value) = 0;
         }
         break;
+    case SLAPI_LDIF_CHANGELOG:
+        if (pblock->pb_task != NULL) {
+            (*(int *)value) = pblock->pb_task->ldif_include_changelog;
+        } else {
+            (*(int *)value) = 0;
+        }
+        break;
     /* dbverify */
     case SLAPI_DBVERIFY_DBDIR:
         if (pblock->pb_task != NULL) {
@@ -3171,6 +3190,18 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value)
         }
         pblock->pb_plugin->plg_bepostbackup = (IFP)value;
         break;
+    case SLAPI_PLUGIN_BE_POST_EXPORT_FN:
+        if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) {
+            return (-1);
+        }
+        pblock->pb_plugin->plg_bepostexport = (IFP)value;
+        break;
+    case SLAPI_PLUGIN_BE_POST_IMPORT_FN:
+        if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) {
+            return (-1);
+        }
+        pblock->pb_plugin->plg_bepostimport = (IFP)value;
+        break;
 
     /* internal preoperation plugin */
     case SLAPI_PLUGIN_INTERNAL_PRE_MODIFY_FN:
@@ -3880,6 +3911,10 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value)
         _pblock_assert_pb_task(pblock);
         pblock->pb_task->import_state = *((int *)value);
         break;
+    case SLAPI_LDIF_CHANGELOG:
+        _pblock_assert_pb_task(pblock);
+        pblock->pb_task->ldif_include_changelog = *((int *)value);
+        break;
 
     case SLAPI_LDIF2DB_ENCRYPT:
     case SLAPI_DB2LDIF_DECRYPT:
diff --git a/ldap/servers/slapd/pblock_v3.h b/ldap/servers/slapd/pblock_v3.h
index 7ec2f37..ffcdf7c 100644
--- a/ldap/servers/slapd/pblock_v3.h
+++ b/ldap/servers/slapd/pblock_v3.h
@@ -60,6 +60,7 @@ typedef struct _slapi_pblock_task
     char *ldif_namespaceid; /* used for name based uniqueid generation */
     int ldif_dump_replica;
     int ldif_dump_uniqueid;     /* dump uniqueid during db2ldif */
+    int ldif_include_changelog;     /* include changelog for import/export */
     int ldif_generate_uniqueid; /* generate uniqueid during db2ldif */
     int ldif_encrypt;           /* used to enable encrypt/decrypt on import and export */
     int seq_type;
diff --git a/ldap/servers/slapd/plugin.c b/ldap/servers/slapd/plugin.c
index b175f74..5b02193 100644
--- a/ldap/servers/slapd/plugin.c
+++ b/ldap/servers/slapd/plugin.c
@@ -388,6 +388,8 @@ plugin_call_plugins(Slapi_PBlock *pb, int whichfunction)
     case SLAPI_PLUGIN_BE_POST_DELETE_FN:
     case SLAPI_PLUGIN_BE_POST_OPEN_FN:
     case SLAPI_PLUGIN_BE_POST_BACKUP_FN:
+    case SLAPI_PLUGIN_BE_POST_EXPORT_FN:
+    case SLAPI_PLUGIN_BE_POST_IMPORT_FN:
         plugin_list_number = PLUGIN_LIST_BEPOSTOPERATION;
         do_op = 1; /* always allow backend callbacks (even during startup) */
         break;
@@ -3625,6 +3627,8 @@ plugin_invoke_plugin_pb(struct slapdplugin *plugin, int operation, Slapi_PBlock
         operation == SLAPI_PLUGIN_BE_PRE_CLOSE_FN ||
         operation == SLAPI_PLUGIN_BE_POST_OPEN_FN ||
         operation == SLAPI_PLUGIN_BE_PRE_BACKUP_FN ||
+        operation == SLAPI_PLUGIN_BE_POST_EXPORT_FN ||
+        operation == SLAPI_PLUGIN_BE_POST_IMPORT_FN ||
         operation == SLAPI_PLUGIN_BE_POST_BACKUP_FN)
         return PR_TRUE;
 
diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h
index 70605f9..1e59b0b 100644
--- a/ldap/servers/slapd/slap.h
+++ b/ldap/servers/slapd/slap.h
@@ -1175,6 +1175,8 @@ struct slapdplugin
             IFP plg_un_bepost_delete; /* delete */
             IFP plg_un_bepost_open;   /* open */
             IFP plg_un_bepost_backup; /* backup */
+            IFP plg_un_bepost_import; /* import */
+            IFP plg_un_bepost_export; /* export */
         } plg_un_bepost;
 #define plg_bepostmodify plg_un.plg_un_bepost.plg_un_bepost_modify
 #define plg_bepostmodrdn plg_un.plg_un_bepost.plg_un_bepost_modrdn
@@ -1182,6 +1184,8 @@ struct slapdplugin
 #define plg_bepostdelete plg_un.plg_un_bepost.plg_un_bepost_delete
 #define plg_bepostopen plg_un.plg_un_bepost.plg_un_bepost_open
 #define plg_bepostbackup plg_un.plg_un_bepost.plg_un_bepost_backup
+#define plg_bepostimport plg_un.plg_un_bepost.plg_un_bepost_import
+#define plg_bepostexport plg_un.plg_un_bepost.plg_un_bepost_export
 
         /* internal  pre-operation plugin structure */
         struct plg_un_internal_pre_operation
diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h
index 0559af2..d82c79a 100644
--- a/ldap/servers/slapd/slapi-plugin.h
+++ b/ldap/servers/slapd/slapi-plugin.h
@@ -7109,6 +7109,8 @@ typedef struct slapi_plugindesc
 #define SLAPI_PLUGIN_BE_POST_DELETE_FN 553
 #define SLAPI_PLUGIN_BE_POST_OPEN_FN   554
 #define SLAPI_PLUGIN_BE_POST_BACKUP_FN 555
+#define SLAPI_PLUGIN_BE_POST_EXPORT_FN 556
+#define SLAPI_PLUGIN_BE_POST_IMPORT_FN 557
 
 /* postoperation plugin to the backend - just before transaction commit */
 #define SLAPI_PLUGIN_BE_TXN_POST_ADD_FN    560
@@ -7403,6 +7405,7 @@ typedef struct slapi_plugindesc
 #define SLAPI_DB2LDIF_FILE 184
 /* dump uniqueid */
 #define SLAPI_DB2LDIF_DUMP_UNIQUEID  176
+#define SLAPI_LDIF_CHANGELOG  1761
 #define SLAPI_DB2LDIF_SERVER_RUNNING 197
 
 /* db2ldif/ldif2db/bak2db/db2bak arguments */
-- 
2.9.5

