sipwise
/
ngcpcfg
mirror of https://github.com/sipwise/ngcpcfg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

TT#17650 Switch to use YAML::XS instead of YAML::Tiny

Browse Source 
The latter does not support YAML 1.1, nor many parts of the
specification. Use the more compliant implementation, in addition to try
to converge to a single one, so that we do not get serialization delta
surprises.

Change-Id: Ie51f1c79859d40ef0877fc0ab75f86ee72e14ea4
changes/68/13868/10
Guillem Jover 9 years ago committed by Alexander Lutay
parent b22e72cc88
commit a4dd0365ef
18 changed files with 136 additions and 143 deletions
Show Stats Download Patch File Download Diff File

2
debian/control vendored
Unescape View File

@ -23,7 +23,6 @@ Build-Depends: asciidoc,
libsys-hostname-long-perl,
libtemplate-perl,
libyaml-libyaml-perl,
libyaml-tiny-perl,
netcat,
pkwalify,
python3-pytest,
@ -53,7 +52,6 @@ Depends: etckeeper,
libsys-hostname-long-perl,
libtemplate-perl,
libyaml-libyaml-perl,
libyaml-tiny-perl,
netcat | netcat-traditional,
ngcp-system-tools,
pkwalify,

8
helper/fileformat_version
Unescape View File

@ -5,10 +5,10 @@
use strict;
use warnings;
use YAML::Tiny;
my $yaml = YAML::Tiny->new;
$yaml = YAML::Tiny->read("/etc/ngcp-config/config.yml");
my $fileformat = $yaml->[0]->{internal}->{fileformat};
use YAML::XS;
my $yaml = YAML::XS::LoadFile('/etc/ngcp-config/config.yml');
my $fileformat = $yaml->{internal}->{fileformat};
if (defined $fileformat) {
print "$fileformat\n";

9
helper/sort-yml
Unescape View File

@ -4,15 +4,14 @@
use strict;
use warnings;
use YAML::Tiny;
my $yaml = YAML::Tiny->new;
use YAML::XS;
my $inputfile = shift or die 'You did not specify an input file name';
my $outputfile = shift or die 'You did not specify an ouput file name';
$yaml = YAML::Tiny->read($inputfile);
my $yaml = YAML::XS::LoadFile($inputfile);
open(my $fh, '>', "$outputfile") or die "Could not open $outputfile for writing";
print $fh $yaml->write_string() or die "Could not write YAML to $outputfile";
YAML::XS::DumpFile($outputfile, $yaml);
## END OF FILE #################################################################

2
helper/sync-db
Unescape View File

@ -18,7 +18,7 @@ my $tt = Template->new({ ABSOLUTE => 1, RELATIVE => 1 });
my $config = {};
foreach my $file (@ARGV) {
$config = merge($config, LoadFile($file) || die $ERRNO);
$config = merge($config, LoadFile($file));
}
# rw connection to central node
my $dbhost = $config->{database}->{central}->{dbhost};

2
helper/tt2-daemon
Unescape View File

@ -76,7 +76,7 @@ sub handle_connections {
$loaded_ymls{$file} = undef;
print $client "Loading $file in memory:" unless $quiet;
my $hm = Hash::Merge->new('RIGHT_PRECEDENT');
$config = $hm->merge( $config, LoadFile($file) || die $ERRNO);
$config = $hm->merge( $config, LoadFile($file));
print $client " OK \n" unless $quiet;
};
};

11
helper/validate-yml
Unescape View File

@ -4,14 +4,15 @@
use strict;
use warnings;
use YAML::Tiny;
use YAML::XS;
use File::Temp qw/tempfile/;
my $yaml = YAML::Tiny->new;
my $inputfile = shift or die 'You did not specify an input file name';
my $outputfile = new File::Temp( UNLINK => 1 );
$yaml = YAML::Tiny->read($inputfile) || die "Could not read $inputfile\n".$yaml->errstr()."\n";
open(my $fh, '>', "$outputfile") or die "Could not open $outputfile for writing";
print $fh $yaml->write_string() or die "Could not write YAML to $outputfile";
my $yaml = YAML::XS::LoadFile($inputfile);
YAML::XS::DumpFile($outputfile, $yaml);
## END OF FILE #################################################################

81
sbin/ngcp-network
Unescape View File

@ -17,7 +17,7 @@ use Regexp::IPv6 qw($IPv6_re);
use Socket;
use Storable qw(dclone);
use Sys::Hostname;
use YAML::Tiny;
use YAML::XS;
our $VERSION = 'UNRELEASED';
@ -149,9 +149,9 @@ foreach my $opt (
# }}}
my $yaml = YAML::Tiny->new;
logger("reading input file $inputfile");
$yaml = YAML::Tiny->read($inputfile)
my $yaml = YAML::XS::LoadFile($inputfile)
or croak "File $inputfile could not be read";
if ( -e "$outputfile" && !-w "$outputfile" ) {
@ -159,7 +159,7 @@ if ( -e "$outputfile" && !-w "$outputfile" ) {
}
my $spce;
if ( defined $yaml->[0]->{hosts}->{self} ) {
if ( defined $yaml->{hosts}->{self} ) {
logger('host "self" identified and set, assuming spce system');
$host = 'self';
$spce = 1;
@ -260,24 +260,24 @@ sub set_interface {
if ( defined $settings->{$k} ) {
if ( $settings->{$k} =~ /^none$/msx ) {
logger("unsetting entry $k");
undef $yaml->[0]->{hosts}->{$host}->{$iface}->{$k};
undef $yaml->{hosts}->{$host}->{$iface}->{$k};
}
elsif ( $settings->{$k} =~ /^delete$/msx ) {
logger("deleting entry $k");
delete $yaml->[0]->{hosts}->{$host}->{$iface}->{$k};
delete $yaml->{hosts}->{$host}->{$iface}->{$k};
}
else {
if($k eq 'shared_ip' || $k eq 'shared_v6ip' || $k eq 'advertised_ip') {
if (my ($matched) = grep { $_ eq $settings->{$k} } @{ $yaml->[0]->{hosts}->{$host}->{$iface}->{$k} }) {
if (my ($matched) = grep { $_ eq $settings->{$k} } @{ $yaml->{hosts}->{$host}->{$iface}->{$k} }) {
logger("not setting $k to $matched to avoid duplicates");
} else {
logger("adding IP entry $k: $settings->{$k}");
push @{ $yaml->[0]->{hosts}->{$host}->{$iface}->{$k} }, $settings->{$k}
push @{ $yaml->{hosts}->{$host}->{$iface}->{$k} }, $settings->{$k}
}
}
else {
logger("adding entry $k: $settings->{$k}");
$yaml->[0]->{hosts}->{$host}->{$iface}->{$k} = $settings->{$k};
$yaml->{hosts}->{$host}->{$iface}->{$k} = $settings->{$k};
}
}
}
@ -298,10 +298,10 @@ sub set_interface {
}
# add interface to list of available interfaces
my $ifaces = $yaml->[0]->{hosts}->{$host}->{interfaces};
my $ifaces = $yaml->{hosts}->{$host}->{interfaces};
if ( !defined $ifaces ) {
logger("no interfaces defined yet, adding interface $iface");
$yaml->[0]->{hosts}->{$host}->{interfaces}->[0] = "$iface";
$yaml->{hosts}->{$host}->{interfaces}->[0] = "$iface";
}
else {
logger("interface = $iface");
@ -321,9 +321,9 @@ sub set_interface {
sub remove_interface {
my $rem_iface = shift or croak 'Usage: remove_interface <interface>';
delete $yaml->[0]->{hosts}->{$host}->{$rem_iface};
delete $yaml->{hosts}->{$host}->{$rem_iface};
my $ifaces = $yaml->[0]->{hosts}->{$host}->{interfaces};
my $ifaces = $yaml->{hosts}->{$host}->{interfaces};
if ( defined $ifaces ) {
logger("removing interface @$ifaces as requested");
undef @{$ifaces};
@ -335,8 +335,8 @@ sub set_yml_config {
my $setting = shift or croak 'Usage: set_yml_config <setting> <value>';
my $value = shift or croak 'Usage: set_yml_config <setting> <value>';
$yaml->[0]->{hosts}->{$host}->{$setting} = "$value";
logger("\$yaml->[0]->{hosts}->{\$host}->{$setting} = $value");
$yaml->{hosts}->{$host}->{$setting} = "$value";
logger("\$yaml->{hosts}->{\$host}->{$setting} = $value");
return;
}
@ -349,14 +349,14 @@ sub set_iface_config {
or croak 'Usage: set_iface_config <iface> <setting> <value>';
if ( any { /^${value}$/msx }
@{ $yaml->[0]->{hosts}->{$host}->{$iface}->{$setting} } )
@{ $yaml->{hosts}->{$host}->{$iface}->{$setting} } )
{
logger("$value for $setting on $iface already defined in host $host");
}
else {
push @{ $yaml->[0]->{hosts}->{$host}->{$iface}->{$setting} }, $value;
push @{ $yaml->{hosts}->{$host}->{$iface}->{$setting} }, $value;
logger(
"\$yaml->[0]->{hosts}->{\$host}->{$iface}->{$setting} => $value");
"\$yaml->{hosts}->{\$host}->{$iface}->{$setting} => $value");
}
return;
@ -365,13 +365,13 @@ sub set_iface_config {
sub set_role {
my $new_role = shift or croak 'Usage: set_role <value>';
if ( any { /^${new_role}$/msx } @{ $yaml->[0]->{hosts}->{$host}->{role} } )
if ( any { /^${new_role}$/msx } @{ $yaml->{hosts}->{$host}->{role} } )
{
logger("role $new_role already defined in host $host");
}
else {
push @{ $yaml->[0]->{hosts}->{$host}->{role} }, $new_role;
logger("\$yaml->[0]->{hosts}->{\$host}->{role} => $new_role");
push @{ $yaml->{hosts}->{$host}->{role} }, $new_role;
logger("\$yaml->{hosts}->{\$host}->{role} => $new_role");
}
return;
@ -381,9 +381,9 @@ sub set_dbnode {
my $new_dbnode = shift;
my @nodes = ( 0, );
foreach my $h (keys %{ $yaml->[0]->{hosts} }) {
if($h ne $host && defined $yaml->[0]->{hosts}->{$h}->{dbnode}) {
push @nodes, $yaml->[0]->{hosts}->{$h}->{dbnode};
foreach my $h (keys %{ $yaml->{hosts} }) {
if($h ne $host && defined $yaml->{hosts}->{$h}->{dbnode}) {
push @nodes, $yaml->{hosts}->{$h}->{dbnode};
}
}
# if no value is given we have 0
@ -421,14 +421,14 @@ sub clone_settings {
logger("clone from = $from");
logger("clone to = $to");
if ( !defined $yaml->[0]->{hosts}->{$from} ) {
if ( !defined $yaml->{hosts}->{$from} ) {
croak "Host '$from' doesn\'t exist, refusing to clone settings.";
}
$yaml->[0]->{hosts}->{$to} = dclone($yaml->[0]->{hosts}->{$from});
$yaml->{hosts}->{$to} = dclone($yaml->{hosts}->{$from});
# adjust peer on-the-fly to prevent user mistakes
if ( defined $yaml->[0]->{hosts}->{$from}->{peer} ) {
if ( defined $yaml->{hosts}->{$from}->{peer} ) {
if ( $to !~ /[ab]$/ ) {
carp "Target host $from not ending with 'a' or 'b', skipping automatic peer config.";
@ -442,7 +442,7 @@ sub clone_settings {
$other_node = $to =~ s/b$/a/r;
}
print "Setting peer for host '$to' to '$other_node'.\n";
$yaml->[0]->{hosts}->{$to}->{peer} = $other_node;
$yaml->{hosts}->{$to}->{peer} = $other_node;
}
}
@ -480,12 +480,12 @@ sub move_settings {
# get rid of the entry from the old section
my @tmp =
grep { !/^${role}$/msx }
@{ $yaml->[0]->{hosts}->{$from}->{role} };
$yaml->[0]->{hosts}->{$from}->{role} = [];
push @{ $yaml->[0]->{hosts}->{$from}->{role} }, @tmp;
@{ $yaml->{hosts}->{$from}->{role} };
$yaml->{hosts}->{$from}->{role} = [];
push @{ $yaml->{hosts}->{$from}->{role} }, @tmp;
# add it to its new place
push @{ $yaml->[0]->{hosts}->{$to}->{role} }, $role;
push @{ $yaml->{hosts}->{$to}->{role} }, $role;
}
}
@ -496,18 +496,18 @@ sub move_settings {
# get rid of the entry from the old section
my @tmp =
grep { !/^${type}$/msx }
@{ $yaml->[0]->{hosts}->{$host}->{$from}->{type} };
$yaml->[0]->{hosts}->{$host}->{$from}->{type} = [];
push @{ $yaml->[0]->{hosts}->{$host}->{$from}->{type} }, @tmp;
@{ $yaml->{hosts}->{$host}->{$from}->{type} };
$yaml->{hosts}->{$host}->{$from}->{type} = [];
push @{ $yaml->{hosts}->{$host}->{$from}->{type} }, @tmp;
# add it to its new place, but only if not already defined yet
if ( any { /^${type}$/msx }
@{ $yaml->[0]->{hosts}->{$host}->{$to}->{type} } )
@{ $yaml->{hosts}->{$host}->{$to}->{type} } )
{
logger("type $type is already defined on host $host, interface $to");
}
else {
push @{ $yaml->[0]->{hosts}->{$host}->{$to}->{type} }, $type;
push @{ $yaml->{hosts}->{$host}->{$to}->{type} }, $type;
}
}
}
@ -528,7 +528,7 @@ if (@set_interface) {
if (@remove_host) {
foreach my $rhost (@remove_host) {
logger("removing host $rhost");
delete $yaml->[0]->{hosts}->{$rhost};
delete $yaml->{hosts}->{$rhost};
}
}
@ -568,12 +568,9 @@ if ( defined $dbnode && ( !defined $move_from || !defined $move_to ) )
move_settings( $move_from, $move_to );
clone_settings ( $clone_from, $clone_to );
open my $fh, '>', "$outputfile"
or croak "Could not open $outputfile for writing";
logger("writing output file $outputfile");
print {$fh} $yaml->write_string()
YAML::XS::DumpFile($outputfile, $yaml)
or croak "Could not write YAML to $outputfile";
close $fh or croak "Couldn't close '$fh': $OS_ERROR";
# }}}

8
sbin/ngcp-sync-constants
Unescape View File

@ -7,7 +7,7 @@ use warnings;
use English qw( -no_match_vars );
use DBI qw(:sql_types);
use Config::Tiny;
use YAML::Tiny;
use YAML::XS;
use Getopt::Long;
use IPC::System::Simple qw(system capturex);
use Readonly;
@ -238,7 +238,7 @@ sub sync_mysql_data {
my $rc = 0;
my $data = get_user_pass($yml->[0]->{credentials}{mysql});
my $data = get_user_pass($yml->{credentials}{mysql});
# special handling for user sipwise as it is not in constants.yml
push @$data, { user => $DEFAULT_MYSQL_USER,
@ -272,7 +272,7 @@ sub flush_privs {
sub main {
eval {
$yml = YAML::Tiny->read($CONSTANTS_YML);
$yml = YAML::XS::LoadFile($CONSTANTS_YML);
};
die "Can't read constants file: $EVAL_ERROR\n" if $EVAL_ERROR;
@ -321,7 +321,7 @@ sub main {
return if $test_mode;
log_info("writing new passwords into $CONSTANTS_YML ... ");
$yml->write($CONSTANTS_YML);
YAML::XS::DumpFile($CONSTANTS_YML, $yml);
log_info("done");
return;

6
sbin/ngcp-sync-grants
Unescape View File

@ -8,7 +8,7 @@ use English;
use DBI;
use Getopt::Long;
use Config::Tiny;
use YAML::Tiny;
use YAML::XS;
use Readonly;
Readonly my $GRANTS_SCHEMA => '/etc/mysql/grants.yml';
@ -464,9 +464,7 @@ sub flush_privs {
}
sub main {
if (my $yml = YAML::Tiny->read($GRANTS_SCHEMA)) {
$grants = $yml->[0];
}
$grants = YAML::XS::LoadFile($GRANTS_SCHEMA);
my ($dbhost, $dbport);
if (my $db_cfg = Config::Tiny->read($DB_CFG)) {

2
scripts/check
Unescape View File

@ -67,7 +67,7 @@ check_config_syntax() {
log_debug "Checking for valid YAML syntax for $config_files"
for f in $config_files ; do
if [ -r "$f" ] ; then
# use YAML::Tiny for checking
# use YAML::XS for checking
log_debug "Validating main YAML syntax of ${f}"
if ! "${HELPER}/validate-yml" "${f}" 2>/dev/null ; then
log_error "Invalid file syntax in ${f}:"

8
scripts/del
Unescape View File

@ -34,7 +34,7 @@ option="$2"
[ -z "${option}" ] && ( log_error "missing option to set. Exiting." ; exit 1)
log_debug "Deleting option '${option}' from '${file}'"
perl_line="delete \$yaml->[0]->{${option//./\}->\{}};"
perl_line="delete \$yaml->{${option//./\}->\{}};"
log_debug "perl line: ${perl_line}"
tmp=$(mktemp)
@ -43,13 +43,13 @@ log_debug "Temporary perl file: ${tmp}"
cat > "${tmp}" << EOF
use strict;
use warnings;
use YAML::Tiny;
use YAML::XS;
my \$file="${file}";
my \$yaml = YAML::Tiny->read("\$file") or die "YAML file '\$file' could not be loaded";
my \$yaml = YAML::XS::LoadFile("\$file");
${perl_line}
\$yaml->write(\$file) or die "Could not write YAML to \$file";
YAML::XS::DumpFile(\$file, \$yaml);
EOF
log_debug "perl -wCSD \"${tmp}\" || RC=$?"

8
scripts/set
Unescape View File

@ -57,7 +57,7 @@ else
esac
fi
perl_line="\$yaml->[0]->{${option//./\}->\{}}=${value};"
perl_line="\$yaml->{${option//./\}->\{}}=${value};"
log_debug "perl line: ${perl_line}"
tmp=$(mktemp)
@ -66,13 +66,13 @@ log_debug "Temporary perl file: ${tmp}"
cat > "${tmp}" << EOF
use strict;
use warnings;
use YAML::Tiny;
use YAML::XS;
my \$file="${file}";
my \$yaml = YAML::Tiny->read("\$file") or die "YAML file '\$file' could not be loaded";
my \$yaml = YAML::XS::LoadFile("\$file");
${perl_line}
\$yaml->write(\$file) or die "Could not write YAML to \$file";
YAML::XS::DumpFile(\$file, \$yaml);
EOF
log_debug "perl -wCSD \"${tmp}\" || RC=$?"

4
t/Dockerfile
Unescape View File

@ -5,12 +5,12 @@ FROM docker.mgm.sipwise.com/sipwise-jessie:latest
# is updated with the current date. It will force refresh of all
# of the base images and things like `apt-get update` won't be using
# old cached versions when the Dockerfile is built.
ENV REFRESHED_AT 2017-05-27
ENV REFRESHED_AT 2017-06-21
RUN apt-get update
# sourcecode test dependencies
RUN apt-get install --assume-yes libyaml-tiny-perl netcat libhash-merge-perl libtemplate-perl libyaml-perl
RUN apt-get install --assume-yes netcat libhash-merge-perl libtemplate-perl libyaml-libyaml-perl libyaml-tiny-perl libyaml-perl
# misc stuff for execution/debugging/devel
RUN apt-get install --assume-yes fakeroot git lsof make strace
# test execution

18
t/test_ngcpcfg_set_and_del.py
Unescape View File

@ -183,10 +183,10 @@ def test_set_action_generate_list(ngcpcfgcli, tmpdir):
assert tmpfile.read() == '''---
aaa:
bbb:
- ccc
- '123'
- ddd
- '567'
- ccc
- '123'
- ddd
- '567'
'''
assert "" in out.stdout
assert "" in out.stderr
@ -203,13 +203,13 @@ ccc:
out = ngcpcfgcli("set", str(tmpfile), "foo.bar=['ha','hi','he','ho']")
assert tmpfile.read() == '''---
ccc:
pytest-was-here: 'true'
pytest-was-here: true
foo:
bar:
- ha
- hi
- he
- ho
- ha
- hi
- he
- ho
'''
assert "" in out.stdout
assert "" in out.stderr

2
testsuite/cpanfile
Unescape View File

@ -11,4 +11,4 @@ requires 'Pod::Usage';
requires 'Regexp::IPv6';
requires 'Socket';
requires 'Sys::Hostname';
requires 'YAML::Tiny';
requires 'YAML::XS';

62
testsuite/network-config/deployment.yml
Unescape View File

@ -3,8 +3,8 @@ hosts:
sp1:
eth0:
dns_nameservers:
- 2.3.4.5
- 3.4.5.6
- 2.3.4.5
- 3.4.5.6
gateway: 1.2.3.1
hwaddr: de:ad:be:ef:23:42
ip: 1.2.3.4
@ -14,11 +14,11 @@ hosts:
ip: 192.168.255.251
netmask: 255.255.255.248
type:
- ha_int
- ha_int
interfaces:
- lo
- eth0
- eth1
- lo
- eth0
- eth1
lo:
hwaddr: 00:00:00:00:00:00
ip: 127.0.0.1
@ -26,28 +26,28 @@ hosts:
shared_ip: ~
shared_v6ip: ~
type:
- sip_int
- web_ext
- sip_ext
- rtp_ext
- ssh_ext
- mon_ext
- web_int
v6ip: '::1'
- sip_int
- web_ext
- sip_ext
- rtp_ext
- ssh_ext
- mon_ext
- web_int
v6ip: ::1
peer: sp2
role:
- proxy
- lb
- mgmt
- proxy
- lb
- mgmt
sp2:
eth1:
ip: 192.168.255.252
netmask: 255.255.255.248
type:
- ha_int
- ha_int
interfaces:
- lo
- eth1
- lo
- eth1
lo:
hwaddr: 00:00:00:00:00:00
ip: 127.0.0.1
@ -55,16 +55,16 @@ hosts:
shared_ip: ~
shared_v6ip: ~
type:
- sip_int
- web_ext
- sip_ext
- rtp_ext
- ssh_ext
- mon_ext
- web_int
v6ip: '::1'
- sip_int
- web_ext
- sip_ext
- rtp_ext
- ssh_ext
- mon_ext
- web_int
v6ip: ::1
peer: sp1
role:
- proxy
- lb
- mgmt
- proxy
- lb
- mgmt

24
testsuite/network-config/eth42.yml
Unescape View File

@ -5,20 +5,20 @@ hosts:
ip: 1.2.3.4
netmask: 255.255.255.248
interfaces:
- lo
- eth42
- lo
- eth42
lo:
ip: 127.0.0.1
netmask: 255.255.255.0
type:
- sip_int
- ha_int
- web_ext
- sip_ext
- rtp_ext
- ssh_ext
- mon_ext
- sip_int
- ha_int
- web_ext
- sip_ext
- rtp_ext
- ssh_ext
- mon_ext
role:
- proxy
- lb
- mgmt
- proxy
- lb
- mgmt

22
testsuite/network-config/network_pro.yml
Unescape View File

@ -2,19 +2,19 @@
hosts:
sp1:
interfaces:
- lo
- lo
lo:
ip: 127.0.0.1
netmask: 255.255.255.0
type:
- sip_int
- ha_int
- web_ext
- sip_ext
- rtp_ext
- ssh_ext
- mon_ext
- sip_int
- ha_int
- web_ext
- sip_ext
- rtp_ext
- ssh_ext
- mon_ext
role:
- proxy
- lb
- mgmt
- proxy
- lb
- mgmt

Write Preview
Loading…
Cancel
Save