Skip to content

Commit 0cc6c6a

Browse files
author
Marcin Przepiorowski
committed
v2.4.23
2 parents 92c6896 + 2f79ecd commit 0cc6c6a

14 files changed

+1131
-53
lines changed

‎CHANGELOG.md

+6
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
## 2.4.23
2+
3+
### Added
4+
- Support for Postgresql ( dSource / VDB )
5+
- Data patch flag added for Oracle VDBs
6+
17
## 2.4.22.1
28

39
### Changes

‎bin/dx_ctl_dsource.pl

+100-4
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,17 @@
8989
'uniqname=s' => \(my $uniqname),
9090
'template=s' => \(my $template),
9191
'oracledbtype=s' => \(my $oracledbtype),
92+
'sourcehostname=s' => \(my $sourcehostname),
93+
'sourceport=n' => \(my $sourceport),
94+
'ingestiontype=s' => \(my $ingestiontype),
95+
'singledbname=s' => \(my $singledbname),
96+
'stagingport=n' => \(my $stagingport),
97+
'dumpdir=s' => \(my $dumpdir),
98+
'restorejobs=n' => \(my $restorejobs),
99+
'dumpjobs=n' => \(my $dumpjobs),
100+
'customparameters=s@' => \(my $customparameters),
101+
'backup_dir_log=s' => \(my $backup_dir_log),
102+
'keepinsync=s' => \(my $keepinsync),
92103
'dever=s' => \(my $dever),
93104
'debug:n' => \(my $debug),
94105
'all' => (\my $all),
@@ -134,13 +145,13 @@
134145
exit (1);
135146
}
136147

137-
if ( defined ($type) && ( ! ( ( lc $type eq 'oracle') || ( lc $type eq 'sybase') || ( lc $type eq 'mssql') || ( lc $type eq 'vfiles') || ( lc $type eq 'db2') ) ) ) {
148+
if ( defined ($type) && ( ! ( ( lc $type eq 'oracle') || ( lc $type eq 'sybase') || ( lc $type eq 'mssql') || ( lc $type eq 'vfiles') || ( lc $type eq 'db2') || ( lc $type eq 'postgresql') ) ) ) {
138149
print "Option -type has invalid parameter - $type \n";
139150
pod2usage(-verbose => 1, -input=>\*DATA);
140151
exit (1);
141152
}
142153

143-
if (((lc $type eq 'vfiles') || (lc $type eq 'db2')) && (lc $action eq 'attach')) {
154+
if (((lc $type eq 'vfiles') || ( lc $type eq 'db2') || ( lc $type eq 'postgresql')) && (lc $action eq 'attach')) {
144155
print "Can't attach $type dSource\n";
145156
exit (1);
146157
}
@@ -153,7 +164,7 @@
153164
exit (1);
154165
}
155166
}
156-
elsif ( ( lc $type ne 'db2' ) && ( ! ( defined($type) && defined($sourcename) && defined($dsourcename) && defined($source_os_user) && defined($group) ) ) ) {
167+
elsif ( ( lc $type ne 'postgresql' ) && ( lc $type ne 'db2' ) && ( ! ( defined($type) && defined($sourcename) && defined($dsourcename) && defined($source_os_user) && defined($group) ) ) ) {
157168
print "Options -sourcename, -dsourcename, -group, -source_os_user are required. \n";
158169
pod2usage(-verbose => 1, -input=>\*DATA);
159170
exit (1);
@@ -175,7 +186,7 @@
175186

176187

177188
} else {
178-
if (defined ($type) && ((lc $type eq 'vfiles') || (lc $type eq 'db2') ) && (lc $action eq 'detach')) {
189+
if (defined ($type) && ((lc $type eq 'vfiles') || (lc $type eq 'db2') || ( lc $type eq 'postgresql') ) && (lc $action eq 'detach')) {
179190
print "Can't deattach $type dSource\n";
180191
exit (1);
181192
}
@@ -229,6 +240,12 @@
229240
pod2usage(-verbose => 1, -input=>\*DATA);
230241
exit (1)
231242
}
243+
} elsif (lc $type eq 'postgresql') {
244+
if ( lc $ingestiontype ne 'externalbackup' ) {
245+
print "Options -dbuser and -password are required for non vFiles dsources. \n";
246+
pod2usage(-verbose => 1, -input=>\*DATA);
247+
exit (1);
248+
}
232249
} else {
233250
print "Options -dbuser and -password are required for non vFiles dsources. \n";
234251
pod2usage(-verbose => 1, -input=>\*DATA);
@@ -410,6 +427,17 @@
410427
}
411428
$jobno = $db->addSource($sourcename,$sourceinst,$sourceenv,$source_os_user,$dbuser,$password,$dsourcename,$group,$logsync,$stageenv,$stageinst,$stage_os_user, $backup_dir, $hadr);
412429
}
430+
elsif ($type eq 'postgresql') {
431+
my $db = new PostgresVDB_obj($engine_obj,$debug);
432+
if (addhooks($hooks, $db, $presync, $postsync)) {
433+
$ret = $ret + 1;
434+
last;
435+
}
436+
$jobno = $db->addSource($sourcename,$dbuser,$password,$dsourcename,$group,$logsync,$stageenv,$stageinst,$stage_os_user, $backup_dir,
437+
$sourcehostname, $sourceport, $ingestiontype, $dumpdir, $restorejobs, $dumpjobs, $stagingport, $singledbname, $mountbase,
438+
$customparameters, $backup_dir, $backup_dir_log, $keepinsync);
439+
440+
}
413441

414442
# we are adding only one dSource - so one job
415443
$ret = $ret + Toolkit_helpers::waitForAction($engine_obj, $jobno, "Action completed with success", "There were problems with dSource action");
@@ -508,6 +536,17 @@ =head1 SYNOPSIS
508536
[-postsync [hookname,]template|filename[,OS_shell] ]
509537
[-stagingpush]
510538
[-oracledbtype nonmt|cdb|pdb]
539+
[-customparameters (param_name=value)|(#param_name)]
540+
[-ingestiontype single|initalized|externalbackup ]
541+
[-sourcehostname hostname ]
542+
[-sourceport xxx]
543+
[-singledbname dbname]
544+
[-stagingport xxx ]
545+
[-dumpdir directory]
546+
[-restorejobs x]
547+
[-dumpjobs x]
548+
[-backup_dir_log directory]
549+
[-keepinsync yes|no]
511550
[-debug ]
512551
[-version ]
513552
[-help|? ]
@@ -659,6 +698,40 @@ =head2 dSource arguments
659698
=item B<-hooks path_to_hooks>
660699
Import hooks exported using dx_get_hooks
661700
701+
=item B<-customparameters (param_name=value)|(#param_name)>
702+
Provide a custom parameter for Postgresql. For more then one parameter, use -customparameters multiple times.
703+
To comment an existing variable inside Postgresql VDB, put a # sign before a parameter.
704+
705+
=item B<-ingestiontype single|initalized|externalbackup >
706+
Postgresql dSource ingestion type
707+
708+
=item B<-sourcehostname hostname>
709+
Postgresql dSource source host
710+
711+
=item B<-sourceport xxx>
712+
Postgresql dSource source port
713+
714+
=item B<-singledbname dbname>
715+
Postgresql dSource single database name
716+
717+
=item B<-stagingport xxx>
718+
Postgresql dSource staging port
719+
720+
=item B<-dumpdir directory>
721+
Postgresql dSource single database dump directory
722+
723+
=item B<-restorejobs x>
724+
Postgresql dSource single database number of restore jobs
725+
726+
=item B<-dumpjobs x>
727+
Postgresql dSource single database number of dump joba
728+
729+
=item B<-backup_dir_log>
730+
Location of WAL logs for external Postgresql ingestion
731+
732+
=item B<-keepinsync yes|no->
733+
Keep a dSource in sync using Postgresql replication. If value set to yes, replication parameters are mandatory
734+
662735
=back
663736
664737
=head2 Hooks
@@ -808,6 +881,27 @@ =head1 EXAMPLES
808881
Waiting for all actions to complete. Parent action is ACTION-2919
809882
Action completed with success
810883
884+
Adding a Posgresql dSource using Delphix initialized backup
885+
886+
dx_ctl_dsource -d dxtest -action create -group "Untitled" -creategroup -dsourcename "postdsource" -type postgresql -sourcename "postdsource" -stageinst "Postgres vFiles (15.2)" \
887+
-stageenv "POSTSTG" -stage_os_user "postgres" -mountbase "/mnt/provision/postdsource" -password xxxxxxxxx -stagingport 5433 -ingestiontype initiated \
888+
-dbuser "delphix" -sourcehostname "sourceserver" -sourceport 5432 -customparameters "deadlock_timeout=123s"
889+
Waiting for all actions to complete. Parent action is ACTION-734
890+
Action completed with success
891+
892+
Adding a Posgresql dSource using single database ingestion
893+
894+
dx_ctl_dsource -d dxtest -action create -group "Untitled" -creategroup -dsourcename "postdsource" -type postgresql -sourcename "postdsource_cluster" -stageinst "Postgres vFiles (15.2)" \
895+
-stageenv "POSTSTG" -mountbase "/mnt/provision/pioro" -dbuser "postgres" -password xxxxxxxxx -stagingport 5433 -sourcehostname "sourceserver" \
896+
-ingestiontype single -sourceport 5432 -singledbname "singleDB" -dumpdir "/home/postgres" -restorejobs 2 -dumpjobs 2
897+
898+
Adding a Postgresql dSource using external backup
899+
900+
dx_ctl_dsource -d dxtest -action create -group "Untitled" -creategroup -dsourcename "extbac" -type postgresql -sourcename "extbac" -stageinst "Postgres vFiles (15.2)" \
901+
-stageenv "POSTSTG" -mountbase "/mnt/provision/extbac" -stagingport 5434 -ingestiontype externalbackup \
902+
-backup_dir "/home/postgres/backup" -backup_dir_log "/home/postgres/backup" -keepinsync no
903+
904+
811905
Updating a backup path and validated sync mode for Sybase
812906
813907
dx_ctl_dsource -d Landshark5 -action update -validatedsync ENABLED -backup_dir "/u02/sybase_back" -dsourcename pubs3
@@ -839,4 +933,6 @@ =head1 EXAMPLES
839933
Action completed with success
840934
841935
936+
937+
842938
=cut

‎bin/dx_ctl_env.pl

+1-1
Original file line numberDiff line numberDiff line change
@@ -410,7 +410,7 @@
410410
} elsif (lc $repotype eq 'postgresql') {
411411
my %native_params;
412412
my %plugin_params_hash = (
413-
"prettyName" => $dbname
413+
"name" => $dbname
414414
);
415415
if ($sourceconfig_obj->createSourceConfig('plugin', $repo->{reference}, $dbname, \%native_params, \%plugin_params_hash)) {
416416
print "Can't add Postgresql $dbname \n";

‎bin/dx_get_dsourcesize.pl

+1-1
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@
170170

171171
if ($dbobj->getEnvironmentName() eq 'NA') {
172172
# staging push
173-
$envname = $dbobj->getStagingEnvironment();
173+
$envname = $dbobj->getStagingEnvironmentName();
174174
} else {
175175
$envname = $dbobj->getEnvironmentName();
176176
}

‎bin/dx_provision_vdb.pl

+37-3
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
my $version = $Toolkit_helpers::version;
5252

5353
my $archivelog = 'yes';
54+
my $datapatch = 'no';
5455

5556
GetOptions(
5657
'help|?' => \(my $help),
@@ -122,6 +123,8 @@
122123
'tdeexportsecret=s' => \(my $tdeexportsecret),
123124
'tdecdbpassword=s' => \(my $tdecdbpassword),
124125
'tdekeyid=s' => \(my $tdekeyid),
126+
'customparameters=s@' => \(my $customparameters),
127+
'datapatch=s' => \($datapatch),
125128
'dever=s' => \(my $dever),
126129
'debug:n' => \(my $debug),
127130
'all' => (\my $all),
@@ -171,7 +174,7 @@
171174
}
172175

173176

174-
if ( ! ( ( $type eq 'oracle') || ( $type eq 'mssql') || ( $type eq 'sybase') || ( $type eq 'mysql') ||( $type eq 'db2') || ( $type eq 'vFiles') ) ) {
177+
if ( ! ( ( $type eq 'oracle') || ( $type eq 'mssql') || ( $type eq 'sybase') || ( $type eq 'mysql') ||( $type eq 'db2') || ( $type eq 'vFiles') || ( $type eq 'postgresql') ) ) {
175178
print "Option -type has invalid parameter - $type \n";
176179
pod2usage(-verbose => 1, -input=>\*DATA);
177180
exit (1);
@@ -288,6 +291,8 @@
288291
$db = new DB2VDB_obj($engine_obj,$debug);
289292
} elsif ($type eq 'vFiles') {
290293
$db = new AppDataVDB_obj($engine_obj,$debug);
294+
} elsif ($type eq 'postgresql') {
295+
$db = new PostgresVDB_obj($engine_obj,$debug);
291296
}
292297

293298
# common database code
@@ -554,7 +559,15 @@
554559
$db->setNewDBID();
555560
}
556561

557-
562+
if (defined($datapatch)) {
563+
if ((lc $datapatch eq 'no') || ( lc $datapatch eq 'yes') ) {
564+
$db->setDataPatch($datapatch);
565+
} else {
566+
print "Datapatch argument should be yes or no. VDB won't be created\n" ;
567+
$ret = $ret + 1;
568+
next;
569+
}
570+
}
558571

559572
if ( defined($template) ) {
560573
if ( $db->setTemplate($template) ) {
@@ -688,6 +701,16 @@
688701
}
689702
$db->setName($targetname, $dbname);
690703
$jobno = $db->createVDB($group,$environment,$envinst);
704+
} elsif ($type eq 'postgresql') {
705+
if (! defined($port)) {
706+
print "Port not defined. VDB won't be created.\n";
707+
$ret = $ret + 1;
708+
next;
709+
}
710+
711+
$db->setName($targetname, $dbname);
712+
$jobno = $db->createVDB($group,$environment,$envinst,$mntpoint,$port, $customparameters);
713+
691714
}
692715

693716
if (defined($snapshotpolicy_ref)) {
@@ -778,13 +801,14 @@ =head1 SYNOPSIS
778801
-tdeexportsecret tde_export_secret]
779802
[-tdekeyid tde_key_id]
780803
[-tdecdbpassword tde_cdb_keystore_password]
804+
[-customparameters (param_name=value)|(#param_name)]
781805
[-help] [-debug]
782806
783807
784808
=head1 DESCRIPTION
785809
786810
Provision VDB from a defined source on the defined target environment.
787-
811+
dx_provision_vdb.pl -d dxtest -type postgresql -group "Untitled" -creategroup -sourcename "pioro" -srcgroup "Untitled" -targetname "postvdb" -dbname "postvdb" -environment "marcinposttgt.dlpxdc.co" -envinst "Postgres vFiles (15.2)" -envUser "postgres" -hooks /tmp/postvdb.dbhooks -mntpoint "/mnt/provision/postvdb" -port "5444" -customparameters "deadlock_timeout=123s" -customparameters "#max_connections"
788812
=head1 ARGUMENTS
789813
790814
=head2 Delphix Engine selection - if not specified a default host(s) from dxtools.conf will be used.
@@ -1006,6 +1030,9 @@ =head2 VDB arguments
10061030
=item B<-tdecdbpassword tde_cdb_keystore_password>
10071031
Password for an existing target CDB keystore (this is an optional parameter is password of the CDB is already set)
10081032
1033+
=item B<-customparameters (param_name=value)|(#param_name)>
1034+
Provide a custom parameter for Postgresql. For more then one parameter, use -customparameters multiple times.
1035+
To comment an existing variable inside Postgresql VDB, put a # sign before a parameter.
10091036
10101037
=back
10111038
@@ -1159,4 +1186,11 @@ =head1 EXAMPLES
11591186
Job JOB-158167 finised with state: COMPLETED VDB created.
11601187
11611188
1189+
Provision a Postgresql VDB
1190+
1191+
dx_provision_vdb -d dxtest -type postgresql -group "Untitled" -sourcename "pioro" -targetname "postvdb" -dbname "postvdb" -environment "POSTTGT" -envinst "Postgres vFiles (15.2)" -mntpoint "/mnt/provision/postvdb" -port "5444" -customparameters "deadlock_timeout=123s" -customparameters "#max_connections"
1192+
Starting job - JOB-608
1193+
0 - 7 - 11 - 18 - 75 - 100
1194+
Job JOB-608 finished with state: COMPLETED
1195+
VDB created.
11621196
=cut

‎lib/Bookmark_obj.pm

+2
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,8 @@ sub createBookmark
217217
$bookmark_timeflow_type = 'AppDataTimeflowPoint'
218218
} elsif ($db->getDBType() eq 'vFiles') {
219219
$bookmark_timeflow_type = 'AppDataTimeflowPoint'
220+
} elsif ($db->getDBType() eq 'postgresql') {
221+
$bookmark_timeflow_type = 'AppDataTimeflowPoint'
220222
} else {
221223
print "Can't determine a DB type. Exiting\n";
222224
return 1;

‎lib/Databases.pm

+4-4
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ use MSSQLVDB_obj;
4141
use SybaseVDB_obj;
4242
use AppDataVDB_obj;
4343
use DB2VDB_obj;
44+
use PluginVDB_obj;
45+
use PostgresVDB_obj;
4446
use Toolkit_obj;
4547
use Toolkit_helpers qw (logger);
4648
use Encode qw(decode_utf8);
@@ -136,10 +138,6 @@ sub LoadDBList
136138
{
137139
$db = MSSQLVDB_obj->new($self->{_dlpxObject}, $self->{_debug});
138140
}
139-
elsif ($dbitem->{type} eq 'PgSQLDatabaseContainer' )
140-
{
141-
$db = PostgresVDB_obj->new($self->{_dlpxObject}, $self->{_debug});
142-
}
143141
elsif ($dbitem->{type} eq 'ASEDBContainer' )
144142
{
145143
$db = SybaseVDB_obj->new($self->{_dlpxObject}, $self->{_debug});
@@ -153,6 +151,8 @@ sub LoadDBList
153151
if (defined($dbitem->{toolkit}) && defined($toolkits->getName($dbitem->{toolkit})) ) {
154152
if ($toolkits->getName($dbitem->{toolkit}) eq 'db2db') {
155153
$db = DB2VDB_obj->new($self->{_dlpxObject}, $self->{_debug});
154+
} elsif ($toolkits->getName($dbitem->{toolkit}) eq 'postgres-vsdk') {
155+
$db = PostgresVDB_obj->new($self->{_dlpxObject}, $self->{_debug});
156156
} else {
157157
$db = AppDataVDB_obj->new($self->{_dlpxObject}, $self->{_debug});
158158
}

‎lib/MSSQLVDB_obj.pm

+1-1
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ sub getConfig
116116
$config = join($joinsep,($config, "-recoveryModel $recoveryModel"));
117117
} elsif ($self->getType() eq 'dSource') {
118118
my $staging_user = $self->getStagingUser();
119-
my $staging_env = $self->getStagingEnvironment();
119+
my $staging_env = $self->getStagingEnvironmentName();
120120
my $staging_inst = $self->getStagingInst();
121121

122122
$config = join($joinsep,($config, "-stageinst \"$staging_inst\""));

0 commit comments

Comments
 (0)