# HG changeset patch # User Aleksej Bobylev # Date 1519235297 -7200 # Node ID 90a5eb560fd6469a7b709ea69e557f5a233ad08b # Parent c3c324c07d3ff78e68b652a685dddd98ce6d5cf9 Update packages listed in the LFS book. diff -r c3c324c07d3f -r 90a5eb560fd6 acl/receipt --- a/acl/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/acl/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -3,23 +3,20 @@ PACKAGE="acl" VERSION="2.2.52" CATEGORY="system-tools" -SHORT_DESC="Commands for Manipulating POSIX Access Control Lists" +SHORT_DESC="Commands for manipulating POSIX Access Control Lists" MAINTAINER="rcx@zoominternet.net" LICENSE="LGPL2.1 GPL2" WEB_SITE="http://savannah.nongnu.org/projects/acl/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/acl.html" TARBALL="$PACKAGE-$VERSION.src.tar.gz" -WGET_URL="http://download.savannah.gnu.org/releases/$PACKAGE/$TARBALL" +WGET_URL="http://download.savannah.gnu.org/releases/acl/$TARBALL" BUILD_DEPENDS="attr-dev bash libtool gettext" -BUILD_DEPENDS_arm="attr-dev" SPLIT="acl-dev" -# Rules to configure and make the package. -compile_rules() -{ - rm /bin/rpm +compile_rules() { + rm /bin/rpm 2>/dev/null sed -i -e 's|/@pkg_name@|&-@pkg_version@|' include/builddefs.in sed -i -e "/TABS-1;/a if (x > (TABS-1)) x = (TABS-1);" \ libacl/__acl_to_any_text.c @@ -30,24 +27,23 @@ --libexecdir=/usr/lib \ $CONFIGURE_ARGS && make && - make install install-lib install-dev DIST_ROOT=$DESTDIR + make install install-lib install-dev DIST_ROOT=$DESTDIR || return 1 - mkdir $install/lib + mkdir -p $install/lib mv -v $install/usr/lib/libacl.so.* $install/lib ln -sfv ../../lib/$(readlink $install/usr/lib/libacl.so) \ $install/usr/lib/libacl.so + find $install -type f \( -name '*.so*' -o -name '*.la' \) \ -exec chmod 755 \{\} \; - } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in acl) copy @std DEPENDS="attr" + TAGS="LFS" ;; acl-dev) copy @dev diff -r c3c324c07d3f -r 90a5eb560fd6 attr/receipt --- a/attr/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/attr/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -3,23 +3,21 @@ PACKAGE="attr" VERSION="2.4.47" CATEGORY="system-tools" -SHORT_DESC="Commands for Manipulating Filesystem Extended Attributes" +SHORT_DESC="Commands for manipulating filesystem extended attributes" MAINTAINER="rcx@zoominternet.net" LICENSE="GPL2" WEB_SITE="http://savannah.nongnu.org/projects/attr" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/attr.html" TARBALL="$PACKAGE-$VERSION.src.tar.gz" -WGET_URL="http://download.savannah.gnu.org/releases/$PACKAGE/$TARBALL" +WGET_URL="http://download.savannah.gnu.org/releases/attr/$TARBALL" +BUILD_DEPENDS_arm=" " BUILD_DEPENDS="autoconf automake m4 libtool gettext coreutils-operations" -BUILD_DEPENDS_arm=" " SPLIT="attr-dev" -# Rules to configure and make the package. -compile_rules() -{ - rm /bin/rpm +compile_rules() { + rm /bin/rpm >/dev/null sed -i -e 's|/@pkg_name@|&-@pkg_version@|' include/builddefs.in ./configure \ @@ -27,21 +25,23 @@ --disable-static \ $CONFIGURE_ARGS && make && - make install install-lib install-dev DIST_ROOT=$DESTDIR + make install install-lib install-dev DIST_ROOT=$DESTDIR || return 1 mkdir -p $install/lib mv $install/usr/lib/libattr.so.* $install/lib ln -sf ../../lib/$(readlink $install/usr/lib/libattr.so) \ $install/usr/lib/libattr.so + find $install -type f \( -name '*.so*' -o -name '*.la' \) \ -exec chmod 755 \{\} \; } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in - attr) copy @std ;; - attr-dev) copy @dev ;; + attr) + copy @std + TAGS="LFS" + ;; + *-dev) copy @dev;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 autoconf/receipt --- a/autoconf/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/autoconf/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -6,22 +6,21 @@ SHORT_DESC="A GNU tool for automatically configuring source code" MAINTAINER="pankso@slitaz.org" LICENSE="GPL2" -WEB_SITE="https://www.gnu.org/software/autoconf/autoconf.html" +WEB_SITE="https://www.gnu.org/software/autoconf/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/autoconf.html" +HOST_ARCH="any" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" BUILD_DEPENDS="m4 perl" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std @dev DEPENDS="m4 perl" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 automake/receipt --- a/automake/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/automake/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,33 +1,26 @@ # SliTaz package receipt v2. PACKAGE="automake" -VERSION="1.15" +VERSION="1.15.1" CATEGORY="development" SHORT_DESC="A GNU tool for automatically creating Makefiles" MAINTAINER="pankso@slitaz.org" LICENSE="GPL2" WEB_SITE="https://www.gnu.org/software/automake/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/automake.html" +HOST_ARCH="i486 x86_64" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" BUILD_DEPENDS="autoconf bison flex gfortran" -# Rules to configure and make the package. -compile_rules() -{ - sed -i 's:/\\\${:/\\\$\\{:' bin/automake.in - - ./configure \ - --docdir=/usr/share/doc/automake-$VERSION \ - $CONFIGURE_ARGS && - make && - make install +compile_rules() { + ./configure $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std @dev DEPENDS="autoconf" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 bash/receipt --- a/bash/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/bash/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,14 +1,14 @@ # SliTaz package receipt v2. PACKAGE="bash" -VERSION="4.4.12" +VERSION="4.4.18" BASEVERSION="${VERSION%.*}" CATEGORY="shells" SHORT_DESC="The GNU bourne shell" MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/bash/" -TAGS="shell" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/bash.html" TARBALL="$PACKAGE-$BASEVERSION.tar.gz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" @@ -21,8 +21,7 @@ BUILD_DEPENDS="ncurses-dev readline-dev texinfo gettext patch" SPLIT="bash-dev" -patch_bash() -{ +patch_bash() { for i in $BASHPATCHES; do [ -s "$SOURCES_REPOSITORY/$i" ] || wget -O "$SOURCES_REPOSITORY/$i" \ @@ -33,19 +32,17 @@ } # Rules to compile & install the temporary toolchain. -cook_tmp_toolchain() -{ +cook_tmp_toolchain() { cd $src patch_bash ./configure --without-bash-malloc && make && make install } -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { # Patch and then build. patch_bash + # Skip tests that can not run while cross-compiling. cat > config.cache <"); - break; - } -+ free($2); - } - | expression '+' expression - { -diff -Naur bc-1.06.95.orig/bc/storage.c bc-1.06.95/bc/storage.c ---- bc-1.06.95.orig/bc/storage.c 2006-09-04 21:39:31.000000000 -0500 -+++ bc-1.06.95/bc/storage.c 2014-04-09 13:28:11.770763410 -0500 -@@ -99,6 +99,7 @@ - { - f = &functions[indx]; - f->f_defined = FALSE; -+ f->f_void = FALSE; - f->f_body = (char *) bc_malloc (BC_START_SIZE); - f->f_body_size = BC_START_SIZE; - f->f_code_size = 0; -diff -Naur bc-1.06.95.orig/bc/util.c bc-1.06.95/bc/util.c ---- bc-1.06.95.orig/bc/util.c 2006-09-04 21:39:31.000000000 -0500 -+++ bc-1.06.95/bc/util.c 2014-04-09 13:27:39.841190064 -0500 -@@ -602,8 +602,7 @@ - case FUNCTDEF: - if (id->f_name != 0) - { -- if (namekind != FUNCT) -- free(name); -+ free(name); - /* Check to see if we are redefining a math lib function. */ - if (use_math && namekind == FUNCTDEF && id->f_name <= 6) - id->f_name = next_func++; diff -r c3c324c07d3f -r 90a5eb560fd6 bc/stuff/patches/series --- a/bc/stuff/patches/series Wed Feb 21 18:10:55 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1 +0,0 @@ -bc-1.06.95-memory_leak-1.patch diff -r c3c324c07d3f -r 90a5eb560fd6 binutils/receipt --- a/binutils/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/binutils/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,40 +1,35 @@ # SliTaz package receipt v2. PACKAGE="binutils" -VERSION="2.27" +VERSION="2.29" CATEGORY="development" SHORT_DESC="GNU assembler, linker and binary utilities" MAINTAINER="pankso@slitaz.org" LICENSE="GPL2 LGPL2 GPL3 LGPL3" WEB_SITE="http://www.gnu.org/software/binutils/" -TAGS="assembler linker" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/binutils.html" TARBALL="$PACKAGE-$VERSION.tar.bz2" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" +BUILD_DEPENDS_arm=" " BUILD_DEPENDS="glibc-dev zlib-dev texinfo bison flex gettext-tools" -BUILD_DEPENDS_arm=" " -SPLIT="libbfd" +SPLIT="libbfd binutils" -# Rules to configure and make the package. -compile_rules() -{ - case "$ARCH" in - i?86) ARCH_ARGS="--disable-werror" ;; - esac - - mkdir build; cd build +compile_rules() { + mkdir build + cd build $src/configure \ --enable-gold \ --enable-ld=default \ --enable-plugins \ --enable-shared \ + --disable-werror \ --with-system-zlib \ - $CONFIGURE_ARGS $ARCH_ARGS && + $CONFIGURE_ARGS && make tooldir=/usr && - make tooldir=/usr install + make tooldir=/usr install || return 1 # Create prefixed version of the tools. cd $install/usr/bin @@ -44,27 +39,21 @@ ln -s $TOOLPREFIX$i $i done - # Remove prefix from manpages name - for i in $(find $install/usr/share/man/man1 -type f); do - mv $i ${i/$TOOLPREFIX/} - done - # # Install libiberty (-liberty flag): dependency described in *.la files # install -o root -g root -m 644 $src/include/libiberty.h $install/usr/include } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in - binutils) - copy @std @dev - DEPENDS="glibc zlib flex libbfd" - ;; libbfd) copy libbfd*.so CAT="system-tools|Binary File Descriptor library" DEPENDS="zlib" ;; + binutils) + copy @std @dev @rm + DEPENDS="flex libbfd glibc" + TAGS="LFS assembler linker" + ;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 bison/receipt --- a/bison/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/bison/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,26 +7,20 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/bison/" -TAGS="parser language" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/bison.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" BUILD_DEPENDS="m4 perl libxslt gettext" -# Rules to configure and make the package. -compile_rules() -{ - ./configure \ - --docdir=/usr/share/doc/bison-$VERSION \ - $CONFIGURE_ARGS && - make && make install +compile_rules() { + ./configure $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ - copy bin/ lib/ aclocal/ bison/ +genpkg_rules() { + copy @std @dev + rm -rf $fs/usr/share/doc # one *.h file copied here with @dev DEPENDS="m4" + TAGS="LFS parser language" } diff -r c3c324c07d3f -r 90a5eb560fd6 bzip2/receipt --- a/bzip2/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/bzip2/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,43 +7,45 @@ MAINTAINER="pankso@slitaz.org" LICENSE="BSD" WEB_SITE="http://www.bzip.org/" -TAGS="compression archive" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/bzip2.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="http://www.bzip.org/$VERSION/$TARBALL" SPLIT="bzlib bzip2-apps bzip2-dev" -# Rules to configure and make the package. -compile_rules() -{ - # No configure script, we must used CC, AR and RANLIB set by cook. +compile_rules() { case "$ARCH" in arm*) - make -f Makefile-libbz2_so CC=${CC} AR=${AR} RANLIB=${RANLIB} && - make clean && make CC=${CC} AR=${AR} RANLIB=${RANLIB} ;; + # No configure script, we must used CC, AR and RANLIB set by cook. + make -f Makefile-libbz2_so CC=$CC AR=$AR RANLIB=$RANLIB && + make clean && make CC=$CC AR=$AR RANLIB=$RANLIB ;; *) + # add large-file support + sed -i 's/^CFLAGS=\(.*\)$/CFLAGS=\1 \$(BIGFILES)/' ./Makefile-libbz2_so + make -f Makefile-libbz2_so && make clean && make ;; - esac + esac && - make install + make install || return 1 + + rm $install/usr/bin/bunzip2 $install/usr/bin/bzcat + ln -s bzip2 $install/bin/bunzip2 + ln -s bzip2 $install/bin/bzcat } # Just to be sure when cross-compiling. -testsuite() -{ +testsuite() { readelf -h $src/bzip2-shared } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in bzip2) copy bzip2 bunzip2 bzcat DEPENDS="bzlib" + TAGS="LFS compression archive" ;; bzlib) copy *.so* @@ -51,8 +53,7 @@ CAT="libs|library" ;; bzip2-apps) - copy bin/ - remove_already_packed + copy bin/ @rm DEPENDS="bzlib" CAT="utilities|apps" ;; diff -r c3c324c07d3f -r 90a5eb560fd6 bzip2/stuff/patches/bzip2-1.0.4-bzip2recover.patch --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bzip2/stuff/patches/bzip2-1.0.4-bzip2recover.patch Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,12 @@ +--- bzip2-1.0.4/bzip2recover.c.pom 2007-01-03 03:00:55.000000000 +0100 ++++ bzip2-1.0.4/bzip2recover.c 2007-02-05 11:55:17.000000000 +0100 +@@ -309,7 +309,8 @@ + UInt32 buffHi, buffLo, blockCRC; + Char* p; + +- strcpy ( progName, argv[0] ); ++ strncpy ( progName, argv[0], BZ_MAX_FILENAME-1); ++ progName[BZ_MAX_FILENAME-1]='\0'; + inFileName[0] = outFileName[0] = 0; + + fprintf ( stderr, diff -r c3c324c07d3f -r 90a5eb560fd6 bzip2/stuff/patches/bzip2-1.0.6-fix_install.patch --- a/bzip2/stuff/patches/bzip2-1.0.6-fix_install.patch Wed Feb 21 18:10:55 2018 +0200 +++ b/bzip2/stuff/patches/bzip2-1.0.6-fix_install.patch Wed Feb 21 19:48:17 2018 +0200 @@ -14,7 +14,7 @@ bzlib.o -all: libbz2.a bzip2 bzip2recover test -+all: libbz2.a bzip2 bzip2recover ++all: bzip2 bzip2recover bzip2: libbz2.a bzip2.o $(CC) $(CFLAGS) $(LDFLAGS) -o bzip2 bzip2.o -L. -lbz2 diff -r c3c324c07d3f -r 90a5eb560fd6 bzip2/stuff/patches/bzip2recover-CVE-2016-3189.patch --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bzip2/stuff/patches/bzip2recover-CVE-2016-3189.patch Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,11 @@ +diff -up ./bzip2recover.c.old ./bzip2recover.c +--- ./bzip2recover.c.old 2016-03-22 08:49:38.855620000 +0100 ++++ ./bzip2recover.c 2016-03-30 10:22:27.341430099 +0200 +@@ -458,6 +458,7 @@ Int32 main ( Int32 argc, Char** argv ) + bsPutUChar ( bsWr, 0x50 ); bsPutUChar ( bsWr, 0x90 ); + bsPutUInt32 ( bsWr, blockCRC ); + bsClose ( bsWr ); ++ outFile = NULL; + } + if (wrBlock >= rbCtr) break; + wrBlock++; diff -r c3c324c07d3f -r 90a5eb560fd6 bzip2/stuff/patches/series --- a/bzip2/stuff/patches/series Wed Feb 21 18:10:55 2018 +0200 +++ b/bzip2/stuff/patches/series Wed Feb 21 19:48:17 2018 +0200 @@ -1,2 +1,4 @@ bzip2-1.0.6-install_docs-1.patch -bzip2-1.0.6-fix_install.patch +-p1|bzip2-1.0.6-fix_install.patch +bzip2-1.0.4-bzip2recover.patch +bzip2recover-CVE-2016-3189.patch # https://bugzilla.redhat.com/attachment.cgi?id=1169843 diff -r c3c324c07d3f -r 90a5eb560fd6 check/receipt --- a/check/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/check/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,27 +7,23 @@ MAINTAINER="erjo@slitaz.org" LICENSE="LGPL2.1" WEB_SITE="https://libcheck.github.io/check/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter05/check.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="https://github.com/libcheck/check/releases/download/$VERSION/$TARBALL" SPLIT="check-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure \ --disable-static \ $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in check) copy *.so* ;; - check-dev) copy @dev checkmk ;; + *-dev) copy @dev checkmk ;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 coreutils-multicall/description.txt --- a/coreutils-multicall/description.txt Wed Feb 21 18:10:55 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,108 +0,0 @@ -Coreutils: Core GNU (file, text, shell) utilities. - - * [ exit with the status determined by expression - * base32: base32 encode/decode data and print to standard output - * base64: base64 encode/decode data and print to standard output - * basename: strip directory and suffix from filenames - * cat: concatenate files and print on the standard output - * chcon: change file SELinux security context - * chgrp: change group ownership - * chmod: change file mode bits - * chown: change file owner and group - * chroot: run command or interactive shell with special root directory - * cksum: checksum and count the bytes in a file - * comm: compare two sorted files line by line - * cp: copy files and directories - * csplit: split a file into sections determined by context lines - * cut: remove sections from each line of files - * date: print or set the system date and time - * dd: convert and copy a file - * df: report file system disk space usage - * dir: list directory contents - * dircolors: color setup for ls - * dirname: strip last component from file name - * du: estimate file space usage - * echo: display a line of text - * env: run a program in a modified environment - * expand: convert tabs to spaces - * expr: evaluate expressions - * factor: factor numbers - * false: do nothing, unsuccessfully - * fmt: simple optimal text formatter - * fold: wrap each input line to fit in specified width - * groups: print the groups a user is in - * head: output the first part of files - * hostid: print the numeric identifier for the current host - * id: print real and effective user and group IDs - * install: copy files and set attributes - * join: join lines of two files on a common field - * kill: send signals to processes, or list signals - * link: call the link function to create a link to a file - * ln: make links between files - * logname: print user's login name - * ls: list directory contents - * md5sum: compute and check MD5 message digest - * mkdir: make directories - * mkfifo: make FIFOs (named pipes) - * mknod: make block or character special files - * mktemp: create a temporary file or directory - * mv: move (rename) files - * nice: run a program with modified scheduling priority - * nl: number lines of files - * nohup: run a command immune to hangups, with output to a non-tty - * nproc: print the number of processing units available - * numfmt: convert numbers from/to human-readable strings - * od: dump files in octal and other formats - * paste: merge lines of files - * pathchk: check whether file names are valid or portable - * pinky: lightweight finger - * pr: convert text files for printing - * printenv: print all or part of environment - * printf: format and print data - * ptx: produce a permuted index of file contents - * pwd: print name of current/working directory - * readlink: print resolved symbolic links or canonical file names - * realpath: print the resolved path - * rm: remove files or directories - * rmdir: remove empty directories - * runcon: run command with specified SELinux security context - * seq: print a sequence of numbers - * sha1sum: compute and check SHA1 message digest - * sha224sum: compute and check SHA224 message digest - * sha256sum: compute and check SHA256 message digest - * sha384sum: compute and check SHA384 message digest - * sha512sum: compute and check SHA512 message digest - * shred: overwrite a file to hide its contents, and optionally delete it - * shuf: generate random permutations - * sleep: delay for a specified amount of time - * sort: sort lines of text files - * split: split a file into pieces - * stat: display file or file system status - * stdbuf: run command with modified buffering operations for its standard - streams - * stty: change and print terminal line settings - * sum: checksum and count the blocks in a file - * sync: synchronize cached writes to persistent storage - * tac: concatenate and print files in reverse - * tail: output the last part of files - * tee: read from standard input and write to standard output and files - * test: check file types and compare values - * timeout: run a command with a time limit - * touch: change file timestamps - * tr: translate or delete characters - * true: do nothing, successfully - * truncate: shrink or extend the size of a file to the specified size - * tsort: perform topological sort - * tty: print the file name of the terminal connected to standard input - * uname: print system information - * unexpand: convert spaces to tabs - * uniq: report or omit repeated lines - * unlink: call the unlink function to remove the specified file - * uptime: tell how long the system has been running - * users: print the user names of users currently logged in to the current - host - * vdir: list directory contents - * wc: print newline, word, and byte counts for each file - * who: show who is logged on - * whoami: print effective user ID - * yes: output a string repeatedly until killed diff -r c3c324c07d3f -r 90a5eb560fd6 coreutils-multicall/receipt --- a/coreutils-multicall/receipt Wed Feb 21 18:10:55 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,68 +0,0 @@ -# SliTaz package receipt v2. - -PACKAGE="coreutils-multicall" -VERSION="8.25" -CATEGORY="system-tools" -SHORT_DESC="Utilities for using and setting the basic system" -MAINTAINER="al.bobylev@gmail.com" -LICENSE="GPL3" -WEB_SITE="https://www.gnu.org/software/coreutils/coreutils.html" - -TARBALL="coreutils-$VERSION.tar.xz" -WGET_URL="$GNU_MIRROR/coreutils/$TARBALL" - -SIBLINGS="coreutils" -BUILD_DEPENDS="automake autoconf gettext xz libcap-dev gmp-dev patch" - -# Rules to configure and make the package. -compile_rules() -{ - # Rebuild fixed translations - msgfmt po/nb.po -o po/nb.gmo - msgfmt po/sl.po -o po/sl.gmo - - autoreconf -fi - - FORCE_UNSAFE_CONFIGURE=1 \ - ./configure \ - --enable-single-binary=symlinks \ - $CONFIGURE_ARGS && - make && make install - - # LFS: Move programs to the locations specified by the FHS - mkdir -p \ - $install/bin \ - $install/usr/sbin \ - $install/usr/share/man/man8 - for i in cat chgrp chmod chown cp date dd df echo false ln ls mkdir \ - mknod mv pwd rm rmdir stty sync true uname; do - rm $install/usr/bin/$i - ln -s /usr/bin/coreutils $install/bin/$i - done - rm $install/usr/bin/chroot - ln -s /usr/bin/coreutils $install/usr/sbin/chroot - mv $install/usr/share/man/man1/chroot.1 \ - $install/usr/share/man/man8/chroot.8 - sed -i 's|"1"|"8"|' $install/usr/share/man/man8/chroot.8 -} - -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ - copy @std - DEPENDS="glibc-base acl attr libcap gmp" -} - - # Remove Busybox applets in order to not clash with ("/bin", "/sbin", - # "/usr/bin" and "/usr/sbin" conflicts with the same filename) -post_install_coreutils_multicall() -{ - rm "$1/bin/base64" # for /usr/bin/base64 - rm "$1/bin/kill" # for /usr/bin/kill - rm "$1/bin/mktemp" # for /usr/bin/mktemp - rm "$1/bin/nice" # for /usr/bin/nice - rm "$1/bin/printenv" # for /usr/bin/printenv - rm "$1/bin/sleep" # for /usr/bin/sleep - rm "$1/bin/stat" # for /usr/bin/stat - rm "$1/bin/touch" # for /usr/bin/touch -} diff -r c3c324c07d3f -r 90a5eb560fd6 coreutils-multicall/stuff --- a/coreutils-multicall/stuff Wed Feb 21 18:10:55 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1 +0,0 @@ -../coreutils/stuff \ No newline at end of file diff -r c3c324c07d3f -r 90a5eb560fd6 coreutils/receipt --- a/coreutils/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/coreutils/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,171 +1,239 @@ # SliTaz package receipt v2. PACKAGE="coreutils" -VERSION="8.25" +VERSION="8.29" CATEGORY="meta" SHORT_DESC="Utilities for using and setting the basic system" MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/coreutils/coreutils.html" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter05/coreutils.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" -SIBLINGS="coreutils-multicall" -BUILD_DEPENDS="automake autoconf gettext xz libcap-dev gmp-dev patch" -SPLIT="coreutils-character coreutils-command coreutils-conditions coreutils-\ -context-system coreutils-context-user coreutils-context-working coreutils-\ -directory coreutils-disk coreutils-file-attributes coreutils-file-format \ -coreutils-file-output-full coreutils-file-output-part coreutils-file-sort \ -coreutils-file-special coreutils-file-summarize coreutils-line coreutils-\ -numeric coreutils-operations coreutils-path coreutils-print coreutils-\ -redirection" +BUILD_DEPENDS="automake autoconf gettext xz libcap-dev gmp-dev patch texinfo \ +openssl-dev" +SPLIT="\ +coreutils-character coreutils-command coreutils-conditions \ +coreutils-context-system coreutils-context-user coreutils-context-working \ +coreutils-directory coreutils-disk coreutils-file-attributes \ +coreutils-file-format coreutils-file-output-full coreutils-file-output-part \ +coreutils-file-sort coreutils-file-special coreutils-file-summarize \ +coreutils-line coreutils-numeric coreutils-operations \ +coreutils-path coreutils-print coreutils-redirection \ +coreutils-multicall:multi" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { + case $SET in + multi) SET_ARGS='--enable-single-binary=symlinks';; + *) SET_ARGS='';; + esac # Rebuild fixed translations msgfmt po/nb.po -o po/nb.gmo msgfmt po/sl.po -o po/sl.gmo autoreconf -fi - FORCE_UNSAFE_CONFIGURE=1 ./configure $CONFIGURE_ARGS && - make && make install + FORCE_UNSAFE_CONFIGURE=1 \ + ./configure \ + --with-openssl \ + $SET_ARGS \ + $CONFIGURE_ARGS && + make && make install || return 1 # LFS: Move programs to the locations specified by the FHS mkdir -p \ $install/bin \ $install/usr/sbin \ $install/usr/share/man/man8 - for i in cat chgrp chmod chown cp date dd df echo false ln ls mkdir \ - mknod mv pwd rm rmdir stty sync true uname; do - mv $install/usr/bin/$i $install/bin - done - mv $install/usr/bin/chroot $install/usr/sbin + + while read from to; do + case $SET in + multi) + rm $install$from + # make relative symlinks + case $to in + /bin/*) ln -s ../../usr/bin/coreutils $install$to;; + *) ln -s ../bin/coreutils $install$to;; + esac + ;; + *) + mv $install$from $install$to + ;; + esac + done < - -+/* Get mbstate_t. */ -+# if HAVE_WCHAR_H -+# include -+# endif -+ - /* A 'struct linebuffer' holds a line of text. */ - - struct linebuffer -@@ -28,6 +33,9 @@ struct linebuffer - size_t size; /* Allocated. */ - size_t length; /* Used. */ - char *buffer; -+# if HAVE_WCHAR_H -+ mbstate_t state; -+# endif - }; - - /* Initialize linebuffer LINEBUFFER for use. */ -diff -Naurp coreutils-8.25-orig/src/cut.c coreutils-8.25/src/cut.c ---- coreutils-8.25-orig/src/cut.c 2016-01-13 05:08:59.000000000 -0600 -+++ coreutils-8.25/src/cut.c 2016-02-08 19:07:10.300944616 -0600 -@@ -28,6 +28,11 @@ - #include - #include - #include -+ -+/* Get mbstate_t, mbrtowc(). */ -+#if HAVE_WCHAR_H -+# include -+#endif - #include "system.h" - - #include "error.h" -@@ -38,6 +43,18 @@ - - #include "set-fields.h" - -+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ -+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 -+# undef MB_LEN_MAX -+# define MB_LEN_MAX 16 -+#endif -+ -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif -+ - /* The official name of this program (e.g., no 'g' prefix). */ - #define PROGRAM_NAME "cut" - -@@ -54,6 +71,52 @@ - } \ - while (0) - -+/* Refill the buffer BUF to get a multibyte character. */ -+#define REFILL_BUFFER(BUF, BUFPOS, BUFLEN, STREAM) \ -+ do \ -+ { \ -+ if (BUFLEN < MB_LEN_MAX && !feof (STREAM) && !ferror (STREAM)) \ -+ { \ -+ memmove (BUF, BUFPOS, BUFLEN); \ -+ BUFLEN += fread (BUF + BUFLEN, sizeof(char), BUFSIZ, STREAM); \ -+ BUFPOS = BUF; \ -+ } \ -+ } \ -+ while (0) -+ -+/* Get wide character on BUFPOS. BUFPOS is not included after that. -+ If byte sequence is not valid as a character, CONVFAIL is true. Otherwise false. */ -+#define GET_NEXT_WC_FROM_BUFFER(WC, BUFPOS, BUFLEN, MBLENGTH, STATE, CONVFAIL) \ -+ do \ -+ { \ -+ mbstate_t state_bak; \ -+ \ -+ if (BUFLEN < 1) \ -+ { \ -+ WC = WEOF; \ -+ break; \ -+ } \ -+ \ -+ /* Get a wide character. */ \ -+ CONVFAIL = false; \ -+ state_bak = STATE; \ -+ MBLENGTH = mbrtowc ((wchar_t *)&WC, BUFPOS, BUFLEN, &STATE); \ -+ \ -+ switch (MBLENGTH) \ -+ { \ -+ case (size_t)-1: \ -+ case (size_t)-2: \ -+ CONVFAIL = true; \ -+ STATE = state_bak; \ -+ /* Fall througn. */ \ -+ \ -+ case 0: \ -+ MBLENGTH = 1; \ -+ break; \ -+ } \ -+ } \ -+ while (0) -+ - - /* Pointer inside RP. When checking if a byte or field is selected - by a finite range, we check if it is between CURRENT_RP.LO -@@ -61,6 +124,9 @@ - CURRENT_RP.HI then we make CURRENT_RP to point to the next range pair. */ - static struct field_range_pair *current_rp; - -+/* Length of the delimiter given as argument to -d. */ -+size_t delimlen; -+ - /* This buffer is used to support the semantics of the -s option - (or lack of same) when the specified field list includes (does - not include) the first field. In both of those cases, the entire -@@ -77,15 +143,25 @@ enum operating_mode - { - undefined_mode, - -- /* Output characters that are in the given bytes. */ -+ /* Output bytes that are at the given positions. */ - byte_mode, - -+ /* Output characters that are at the given positions. */ -+ character_mode, -+ - /* Output the given delimiter-separated fields. */ - field_mode - }; - - static enum operating_mode operating_mode; - -+/* If nonzero, when in byte mode, don't split multibyte characters. */ -+static int byte_mode_character_aware; -+ -+/* If nonzero, the function for single byte locale is work -+ if this program runs on multibyte locale. */ -+static int force_singlebyte_mode; -+ - /* If true do not output lines containing no delimiter characters. - Otherwise, all such lines are printed. This option is valid only - with field mode. */ -@@ -97,6 +173,9 @@ static bool complement; - - /* The delimiter character for field mode. */ - static unsigned char delim; -+#if HAVE_WCHAR_H -+static wchar_t wcdelim; -+#endif - - /* The delimiter for each line/record. */ - static unsigned char line_delim = '\n'; -@@ -164,7 +243,7 @@ Print selected parts of lines from each - -f, --fields=LIST select only these fields; also print any line\n\ - that contains no delimiter character, unless\n\ - the -s option is specified\n\ -- -n (ignored)\n\ -+ -n with -b: don't split multibyte characters\n\ - "), stdout); - fputs (_("\ - --complement complement the set of selected bytes, characters\n\ -@@ -280,6 +359,82 @@ cut_bytes (FILE *stream) - } - } - -+#if HAVE_MBRTOWC -+/* This function is in use for the following case. -+ -+ 1. Read from the stream STREAM, printing to standard output any selected -+ characters. -+ -+ 2. Read from stream STREAM, printing to standard output any selected bytes, -+ without splitting multibyte characters. */ -+ -+static void -+cut_characters_or_cut_bytes_no_split (FILE *stream) -+{ -+ size_t idx; /* number of bytes or characters in the line so far. */ -+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ -+ char *bufpos; /* Next read position of BUF. */ -+ size_t buflen; /* The length of the byte sequence in buf. */ -+ wint_t wc; /* A gotten wide character. */ -+ size_t mblength; /* The byte size of a multibyte character which shows -+ as same character as WC. */ -+ mbstate_t state; /* State of the stream. */ -+ bool convfail = false; /* true, when conversion failed. Otherwise false. */ -+ /* Whether to begin printing delimiters between ranges for the current line. -+ Set after we've begun printing data corresponding to the first range. */ -+ bool print_delimiter = false; -+ -+ idx = 0; -+ buflen = 0; -+ bufpos = buf; -+ memset (&state, '\0', sizeof(mbstate_t)); -+ -+ current_rp = frp; -+ -+ while (1) -+ { -+ REFILL_BUFFER (buf, bufpos, buflen, stream); -+ -+ GET_NEXT_WC_FROM_BUFFER (wc, bufpos, buflen, mblength, state, convfail); -+ (void) convfail; /* ignore unused */ -+ -+ if (wc == WEOF) -+ { -+ if (idx > 0) -+ putchar (line_delim); -+ break; -+ } -+ else if (wc == line_delim) -+ { -+ putchar (line_delim); -+ idx = 0; -+ print_delimiter = false; -+ current_rp = frp; -+ } -+ else -+ { -+ next_item (&idx); -+ if (print_kth (idx)) -+ { -+ if (output_delimiter_specified) -+ { -+ if (print_delimiter && is_range_start_index (idx)) -+ { -+ fwrite (output_delimiter_string, sizeof (char), -+ output_delimiter_length, stdout); -+ } -+ print_delimiter = true; -+ } -+ fwrite (bufpos, mblength, sizeof(char), stdout); -+ } -+ } -+ -+ buflen -= mblength; -+ bufpos += mblength; -+ } -+} -+#endif -+ - /* Read from stream STREAM, printing to standard output any selected fields. */ - - static void -@@ -425,13 +580,211 @@ cut_fields (FILE *stream) - } - } - -+#if HAVE_MBRTOWC -+static void -+cut_fields_mb (FILE *stream) -+{ -+ int c; -+ size_t field_idx; -+ int found_any_selected_field; -+ int buffer_first_field; -+ int empty_input; -+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ -+ char *bufpos; /* Next read position of BUF. */ -+ size_t buflen; /* The length of the byte sequence in buf. */ -+ wint_t wc = 0; /* A gotten wide character. */ -+ size_t mblength; /* The byte size of a multibyte character which shows -+ as same character as WC. */ -+ mbstate_t state; /* State of the stream. */ -+ bool convfail = false; /* true, when conversion failed. Otherwise false. */ -+ -+ current_rp = frp; -+ -+ found_any_selected_field = 0; -+ field_idx = 1; -+ bufpos = buf; -+ buflen = 0; -+ memset (&state, '\0', sizeof(mbstate_t)); -+ -+ c = getc (stream); -+ empty_input = (c == EOF); -+ if (c != EOF) -+ { -+ ungetc (c, stream); -+ wc = 0; -+ } -+ else -+ wc = WEOF; -+ -+ /* To support the semantics of the -s flag, we may have to buffer -+ all of the first field to determine whether it is `delimited.' -+ But that is unnecessary if all non-delimited lines must be printed -+ and the first field has been selected, or if non-delimited lines -+ must be suppressed and the first field has *not* been selected. -+ That is because a non-delimited line has exactly one field. */ -+ buffer_first_field = (suppress_non_delimited ^ !print_kth (1)); -+ -+ while (1) -+ { -+ if (field_idx == 1 && buffer_first_field) -+ { -+ int len = 0; -+ -+ while (1) -+ { -+ REFILL_BUFFER (buf, bufpos, buflen, stream); -+ -+ GET_NEXT_WC_FROM_BUFFER -+ (wc, bufpos, buflen, mblength, state, convfail); -+ -+ if (wc == WEOF) -+ break; -+ -+ field_1_buffer = xrealloc (field_1_buffer, len + mblength); -+ memcpy (field_1_buffer + len, bufpos, mblength); -+ len += mblength; -+ buflen -= mblength; -+ bufpos += mblength; -+ -+ if (!convfail && (wc == line_delim || wc == wcdelim)) -+ break; -+ } -+ -+ if (len <= 0 && wc == WEOF) -+ break; -+ -+ /* If the first field extends to the end of line (it is not -+ delimited) and we are printing all non-delimited lines, -+ print this one. */ -+ if (convfail || (!convfail && wc != wcdelim)) -+ { -+ if (suppress_non_delimited) -+ { -+ /* Empty. */ -+ } -+ else -+ { -+ fwrite (field_1_buffer, sizeof (char), len, stdout); -+ /* Make sure the output line is newline terminated. */ -+ if (convfail || (!convfail && wc != line_delim)) -+ putchar (line_delim); -+ } -+ continue; -+ } -+ -+ if (print_kth (1)) -+ { -+ /* Print the field, but not the trailing delimiter. */ -+ fwrite (field_1_buffer, sizeof (char), len - 1, stdout); -+ found_any_selected_field = 1; -+ } -+ next_item (&field_idx); -+ } -+ -+ if (wc != WEOF) -+ { -+ if (print_kth (field_idx)) -+ { -+ if (found_any_selected_field) -+ { -+ fwrite (output_delimiter_string, sizeof (char), -+ output_delimiter_length, stdout); -+ } -+ found_any_selected_field = 1; -+ } -+ -+ while (1) -+ { -+ REFILL_BUFFER (buf, bufpos, buflen, stream); -+ -+ GET_NEXT_WC_FROM_BUFFER -+ (wc, bufpos, buflen, mblength, state, convfail); -+ -+ if (wc == WEOF) -+ break; -+ else if (!convfail && (wc == wcdelim || wc == line_delim)) -+ { -+ buflen -= mblength; -+ bufpos += mblength; -+ break; -+ } -+ -+ if (print_kth (field_idx)) -+ fwrite (bufpos, mblength, sizeof(char), stdout); -+ -+ buflen -= mblength; -+ bufpos += mblength; -+ } -+ } -+ -+ if ((!convfail || wc == line_delim) && buflen < 1) -+ wc = WEOF; -+ -+ if (!convfail && wc == wcdelim) -+ next_item (&field_idx); -+ else if (wc == WEOF || (!convfail && wc == line_delim)) -+ { -+ if (found_any_selected_field -+ || (!empty_input && !(suppress_non_delimited && field_idx == 1))) -+ putchar (line_delim); -+ if (wc == WEOF) -+ break; -+ field_idx = 1; -+ current_rp = frp; -+ found_any_selected_field = 0; -+ } -+ } -+} -+#endif -+ - static void - cut_stream (FILE *stream) - { -- if (operating_mode == byte_mode) -- cut_bytes (stream); -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1 && !force_singlebyte_mode) -+ { -+ switch (operating_mode) -+ { -+ case byte_mode: -+ if (byte_mode_character_aware) -+ cut_characters_or_cut_bytes_no_split (stream); -+ else -+ cut_bytes (stream); -+ break; -+ -+ case character_mode: -+ cut_characters_or_cut_bytes_no_split (stream); -+ break; -+ -+ case field_mode: -+ if (delimlen == 1) -+ { -+ /* Check if we have utf8 multibyte locale, so we can use this -+ optimization because of uniqueness of characters, which is -+ not true for e.g. SJIS */ -+ char * loc = setlocale(LC_CTYPE, NULL); -+ if (loc && (strstr (loc, "UTF-8") || strstr (loc, "utf-8") || -+ strstr (loc, "UTF8") || strstr (loc, "utf8"))) -+ { -+ cut_fields (stream); -+ break; -+ } -+ } -+ cut_fields_mb (stream); -+ break; -+ -+ default: -+ abort (); -+ } -+ } - else -- cut_fields (stream); -+#endif -+ { -+ if (operating_mode == field_mode) -+ cut_fields (stream); -+ else -+ cut_bytes (stream); -+ } - } - - /* Process file FILE to standard output. -@@ -483,6 +836,7 @@ main (int argc, char **argv) - bool ok; - bool delim_specified = false; - char *spec_list_string IF_LINT ( = NULL); -+ char mbdelim[MB_LEN_MAX + 1]; - - initialize_main (&argc, &argv); - set_program_name (argv[0]); -@@ -505,7 +859,6 @@ main (int argc, char **argv) - switch (optc) - { - case 'b': -- case 'c': - /* Build the byte list. */ - if (operating_mode != undefined_mode) - FATAL_ERROR (_("only one type of list may be specified")); -@@ -513,6 +866,14 @@ main (int argc, char **argv) - spec_list_string = optarg; - break; - -+ case 'c': -+ /* Build the character list. */ -+ if (operating_mode != undefined_mode) -+ FATAL_ERROR (_("only one type of list may be specified")); -+ operating_mode = character_mode; -+ spec_list_string = optarg; -+ break; -+ - case 'f': - /* Build the field list. */ - if (operating_mode != undefined_mode) -@@ -524,10 +885,38 @@ main (int argc, char **argv) - case 'd': - /* New delimiter. */ - /* Interpret -d '' to mean 'use the NUL byte as the delimiter.' */ -- if (optarg[0] != '\0' && optarg[1] != '\0') -- FATAL_ERROR (_("the delimiter must be a single character")); -- delim = optarg[0]; -- delim_specified = true; -+ { -+#if HAVE_MBRTOWC -+ if(MB_CUR_MAX > 1) -+ { -+ mbstate_t state; -+ -+ memset (&state, '\0', sizeof(mbstate_t)); -+ delimlen = mbrtowc (&wcdelim, optarg, strnlen(optarg, MB_LEN_MAX), &state); -+ -+ if (delimlen == (size_t)-1 || delimlen == (size_t)-2) -+ ++force_singlebyte_mode; -+ else -+ { -+ delimlen = (delimlen < 1) ? 1 : delimlen; -+ if (wcdelim != L'\0' && *(optarg + delimlen) != '\0') -+ FATAL_ERROR (_("the delimiter must be a single character")); -+ memcpy (mbdelim, optarg, delimlen); -+ mbdelim[delimlen] = '\0'; -+ if (delimlen == 1) -+ delim = *optarg; -+ } -+ } -+ -+ if (MB_CUR_MAX <= 1 || force_singlebyte_mode) -+#endif -+ { -+ if (optarg[0] != '\0' && optarg[1] != '\0') -+ FATAL_ERROR (_("the delimiter must be a single character")); -+ delim = (unsigned char) optarg[0]; -+ } -+ delim_specified = true; -+ } - break; - - case OUTPUT_DELIMITER_OPTION: -@@ -540,6 +929,7 @@ main (int argc, char **argv) - break; - - case 'n': -+ byte_mode_character_aware = 1; - break; - - case 's': -@@ -579,15 +969,34 @@ main (int argc, char **argv) - | (complement ? SETFLD_COMPLEMENT : 0) ); - - if (!delim_specified) -- delim = '\t'; -+ { -+ delim = '\t'; -+#ifdef HAVE_MBRTOWC -+ wcdelim = L'\t'; -+ mbdelim[0] = '\t'; -+ mbdelim[1] = '\0'; -+ delimlen = 1; -+#endif -+ } - - if (output_delimiter_string == NULL) - { -- static char dummy[2]; -- dummy[0] = delim; -- dummy[1] = '\0'; -- output_delimiter_string = dummy; -- output_delimiter_length = 1; -+#ifdef HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1 && !force_singlebyte_mode) -+ { -+ output_delimiter_string = xstrdup(mbdelim); -+ output_delimiter_length = delimlen; -+ } -+ -+ if (MB_CUR_MAX <= 1 || force_singlebyte_mode) -+#endif -+ { -+ static char dummy[2]; -+ dummy[0] = delim; -+ dummy[1] = '\0'; -+ output_delimiter_string = dummy; -+ output_delimiter_length = 1; -+ } - } - - if (optind == argc) -diff -Naurp coreutils-8.25-orig/src/expand.c coreutils-8.25/src/expand.c ---- coreutils-8.25-orig/src/expand.c 2016-01-01 07:48:50.000000000 -0600 -+++ coreutils-8.25/src/expand.c 2016-02-08 19:07:10.301944619 -0600 -@@ -37,12 +37,34 @@ - #include - #include - #include -+ -+/* Get mbstate_t, mbrtowc(), wcwidth(). */ -+#if HAVE_WCHAR_H -+# include -+#endif -+ -+/* Get iswblank(). */ -+#if HAVE_WCTYPE_H -+# include -+#endif -+ - #include "system.h" - #include "error.h" - #include "fadvise.h" - #include "quote.h" - #include "xstrndup.h" - -+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ -+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 -+# define MB_LEN_MAX 16 -+#endif -+ -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif -+ - /* The official name of this program (e.g., no 'g' prefix). */ - #define PROGRAM_NAME "expand" - -@@ -357,6 +379,142 @@ expand (void) - } - } - -+#if HAVE_MBRTOWC -+static void -+expand_multibyte (void) -+{ -+ FILE *fp; /* Input strem. */ -+ mbstate_t i_state; /* Current shift state of the input stream. */ -+ mbstate_t i_state_bak; /* Back up the I_STATE. */ -+ mbstate_t o_state; /* Current shift state of the output stream. */ -+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ -+ char *bufpos = buf; /* Next read position of BUF. */ -+ size_t buflen = 0; /* The length of the byte sequence in buf. */ -+ wchar_t wc; /* A gotten wide character. */ -+ size_t mblength; /* The byte size of a multibyte character -+ which shows as same character as WC. */ -+ int tab_index = 0; /* Index in `tab_list' of next tabstop. */ -+ int column = 0; /* Column on screen of the next char. */ -+ int next_tab_column; /* Column the next tab stop is on. */ -+ int convert = 1; /* If nonzero, perform translations. */ -+ -+ fp = next_file ((FILE *) NULL); -+ if (fp == NULL) -+ return; -+ -+ memset (&o_state, '\0', sizeof(mbstate_t)); -+ memset (&i_state, '\0', sizeof(mbstate_t)); -+ -+ for (;;) -+ { -+ /* Refill the buffer BUF. */ -+ if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp)) -+ { -+ memmove (buf, bufpos, buflen); -+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp); -+ bufpos = buf; -+ } -+ -+ /* No character is left in BUF. */ -+ if (buflen < 1) -+ { -+ fp = next_file (fp); -+ -+ if (fp == NULL) -+ break; /* No more files. */ -+ else -+ { -+ memset (&i_state, '\0', sizeof(mbstate_t)); -+ continue; -+ } -+ } -+ -+ /* Get a wide character. */ -+ i_state_bak = i_state; -+ mblength = mbrtowc (&wc, bufpos, buflen, &i_state); -+ -+ switch (mblength) -+ { -+ case (size_t)-1: /* illegal byte sequence. */ -+ case (size_t)-2: -+ mblength = 1; -+ i_state = i_state_bak; -+ if (convert) -+ { -+ ++column; -+ if (convert_entire_line == 0 && !isblank(*bufpos)) -+ convert = 0; -+ } -+ putchar (*bufpos); -+ break; -+ -+ case 0: /* null. */ -+ mblength = 1; -+ if (convert && convert_entire_line == 0) -+ convert = 0; -+ putchar ('\0'); -+ break; -+ -+ default: -+ if (wc == L'\n') /* LF. */ -+ { -+ tab_index = 0; -+ column = 0; -+ convert = 1; -+ putchar ('\n'); -+ } -+ else if (wc == L'\t' && convert) /* Tab. */ -+ { -+ if (tab_size == 0) -+ { -+ /* Do not let tab_index == first_free_tab; -+ stop when it is 1 less. */ -+ while (tab_index < first_free_tab - 1 -+ && column >= tab_list[tab_index]) -+ tab_index++; -+ next_tab_column = tab_list[tab_index]; -+ if (tab_index < first_free_tab - 1) -+ tab_index++; -+ if (column >= next_tab_column) -+ next_tab_column = column + 1; -+ } -+ else -+ next_tab_column = column + tab_size - column % tab_size; -+ -+ while (column < next_tab_column) -+ { -+ putchar (' '); -+ ++column; -+ } -+ } -+ else /* Others. */ -+ { -+ if (convert) -+ { -+ if (wc == L'\b') -+ { -+ if (column > 0) -+ --column; -+ } -+ else -+ { -+ int width; /* The width of WC. */ -+ -+ width = wcwidth (wc); -+ column += (width > 0) ? width : 0; -+ if (convert_entire_line == 0 && !iswblank(wc)) -+ convert = 0; -+ } -+ } -+ fwrite (bufpos, sizeof(char), mblength, stdout); -+ } -+ } -+ buflen -= mblength; -+ bufpos += mblength; -+ } -+} -+#endif -+ - int - main (int argc, char **argv) - { -@@ -421,7 +579,12 @@ main (int argc, char **argv) - - file_list = (optind < argc ? &argv[optind] : stdin_argv); - -- expand (); -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ expand_multibyte (); -+ else -+#endif -+ expand (); - - if (have_read_stdin && fclose (stdin) != 0) - error (EXIT_FAILURE, errno, "-"); -diff -Naurp coreutils-8.25-orig/src/fold.c coreutils-8.25/src/fold.c ---- coreutils-8.25-orig/src/fold.c 2016-01-01 07:48:50.000000000 -0600 -+++ coreutils-8.25/src/fold.c 2016-02-08 19:07:10.302944622 -0600 -@@ -22,11 +22,33 @@ - #include - #include - -+/* Get mbstate_t, mbrtowc(), wcwidth(). */ -+#if HAVE_WCHAR_H -+# include -+#endif -+ -+/* Get iswprint(), iswblank(), wcwidth(). */ -+#if HAVE_WCTYPE_H -+# include -+#endif -+ - #include "system.h" - #include "error.h" - #include "fadvise.h" - #include "xdectoint.h" - -+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ -+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 -+# undef MB_LEN_MAX -+# define MB_LEN_MAX 16 -+#endif -+ -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif -+ - #define TAB_WIDTH 8 - - /* The official name of this program (e.g., no 'g' prefix). */ -@@ -34,20 +56,41 @@ - - #define AUTHORS proper_name ("David MacKenzie") - -+#define FATAL_ERROR(Message) \ -+ do \ -+ { \ -+ error (0, 0, (Message)); \ -+ usage (2); \ -+ } \ -+ while (0) -+ -+enum operating_mode -+{ -+ /* Fold texts by columns that are at the given positions. */ -+ column_mode, -+ -+ /* Fold texts by bytes that are at the given positions. */ -+ byte_mode, -+ -+ /* Fold texts by characters that are at the given positions. */ -+ character_mode, -+}; -+ -+/* The argument shows current mode. (Default: column_mode) */ -+static enum operating_mode operating_mode; -+ - /* If nonzero, try to break on whitespace. */ - static bool break_spaces; - --/* If nonzero, count bytes, not column positions. */ --static bool count_bytes; -- - /* If nonzero, at least one of the files we read was standard input. */ - static bool have_read_stdin; - --static char const shortopts[] = "bsw:0::1::2::3::4::5::6::7::8::9::"; -+static char const shortopts[] = "bcsw:0::1::2::3::4::5::6::7::8::9::"; - - static struct option const longopts[] = - { - {"bytes", no_argument, NULL, 'b'}, -+ {"characters", no_argument, NULL, 'c'}, - {"spaces", no_argument, NULL, 's'}, - {"width", required_argument, NULL, 'w'}, - {GETOPT_HELP_OPTION_DECL}, -@@ -75,6 +118,7 @@ Wrap input lines in each FILE, writing t - - fputs (_("\ - -b, --bytes count bytes rather than columns\n\ -+ -c, --characters count characters rather than columns\n\ - -s, --spaces break at spaces\n\ - -w, --width=WIDTH use WIDTH columns instead of 80\n\ - "), stdout); -@@ -92,7 +136,7 @@ Wrap input lines in each FILE, writing t - static size_t - adjust_column (size_t column, char c) - { -- if (!count_bytes) -+ if (operating_mode != byte_mode) - { - if (c == '\b') - { -@@ -115,30 +159,14 @@ adjust_column (size_t column, char c) - to stdout, with maximum line length WIDTH. - Return true if successful. */ - --static bool --fold_file (char const *filename, size_t width) -+static void -+fold_text (FILE *istream, size_t width, int *saved_errno) - { -- FILE *istream; - int c; - size_t column = 0; /* Screen column where next char will go. */ - size_t offset_out = 0; /* Index in 'line_out' for next char. */ - static char *line_out = NULL; - static size_t allocated_out = 0; -- int saved_errno; -- -- if (STREQ (filename, "-")) -- { -- istream = stdin; -- have_read_stdin = true; -- } -- else -- istream = fopen (filename, "r"); -- -- if (istream == NULL) -- { -- error (0, errno, "%s", quotef (filename)); -- return false; -- } - - fadvise (istream, FADVISE_SEQUENTIAL); - -@@ -168,6 +196,15 @@ fold_file (char const *filename, size_t - bool found_blank = false; - size_t logical_end = offset_out; - -+ /* If LINE_OUT has no wide character, -+ put a new wide character in LINE_OUT -+ if column is bigger than width. */ -+ if (offset_out == 0) -+ { -+ line_out[offset_out++] = c; -+ continue; -+ } -+ - /* Look for the last blank. */ - while (logical_end) - { -@@ -214,11 +251,221 @@ fold_file (char const *filename, size_t - line_out[offset_out++] = c; - } - -- saved_errno = errno; -+ *saved_errno = errno; -+ -+ if (offset_out) -+ fwrite (line_out, sizeof (char), (size_t) offset_out, stdout); -+ -+} -+ -+#if HAVE_MBRTOWC -+static void -+fold_multibyte_text (FILE *istream, size_t width, int *saved_errno) -+{ -+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ -+ size_t buflen = 0; /* The length of the byte sequence in buf. */ -+ char *bufpos = buf; /* Next read position of BUF. */ -+ wint_t wc; /* A gotten wide character. */ -+ size_t mblength; /* The byte size of a multibyte character which shows -+ as same character as WC. */ -+ mbstate_t state, state_bak; /* State of the stream. */ -+ int convfail = 0; /* 1, when conversion is failed. Otherwise 0. */ -+ -+ static char *line_out = NULL; -+ size_t offset_out = 0; /* Index in `line_out' for next char. */ -+ static size_t allocated_out = 0; -+ -+ int increment; -+ size_t column = 0; -+ -+ size_t last_blank_pos; -+ size_t last_blank_column; -+ int is_blank_seen; -+ int last_blank_increment = 0; -+ int is_bs_following_last_blank; -+ size_t bs_following_last_blank_num; -+ int is_cr_after_last_blank; -+ -+#define CLEAR_FLAGS \ -+ do \ -+ { \ -+ last_blank_pos = 0; \ -+ last_blank_column = 0; \ -+ is_blank_seen = 0; \ -+ is_bs_following_last_blank = 0; \ -+ bs_following_last_blank_num = 0; \ -+ is_cr_after_last_blank = 0; \ -+ } \ -+ while (0) -+ -+#define START_NEW_LINE \ -+ do \ -+ { \ -+ putchar ('\n'); \ -+ column = 0; \ -+ offset_out = 0; \ -+ CLEAR_FLAGS; \ -+ } \ -+ while (0) -+ -+ CLEAR_FLAGS; -+ memset (&state, '\0', sizeof(mbstate_t)); -+ -+ for (;; bufpos += mblength, buflen -= mblength) -+ { -+ if (buflen < MB_LEN_MAX && !feof (istream) && !ferror (istream)) -+ { -+ memmove (buf, bufpos, buflen); -+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, istream); -+ bufpos = buf; -+ } -+ -+ if (buflen < 1) -+ break; -+ -+ /* Get a wide character. */ -+ state_bak = state; -+ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &state); -+ -+ switch (mblength) -+ { -+ case (size_t)-1: -+ case (size_t)-2: -+ convfail++; -+ state = state_bak; -+ /* Fall through. */ -+ -+ case 0: -+ mblength = 1; -+ break; -+ } -+ -+rescan: -+ if (operating_mode == byte_mode) /* byte mode */ -+ increment = mblength; -+ else if (operating_mode == character_mode) /* character mode */ -+ increment = 1; -+ else /* column mode */ -+ { -+ if (convfail) -+ increment = 1; -+ else -+ { -+ switch (wc) -+ { -+ case L'\n': -+ fwrite (line_out, sizeof(char), offset_out, stdout); -+ START_NEW_LINE; -+ continue; -+ -+ case L'\b': -+ increment = (column > 0) ? -1 : 0; -+ break; -+ -+ case L'\r': -+ increment = -1 * column; -+ break; -+ -+ case L'\t': -+ increment = 8 - column % 8; -+ break; -+ -+ default: -+ increment = wcwidth (wc); -+ increment = (increment < 0) ? 0 : increment; -+ } -+ } -+ } -+ -+ if (column + increment > width && break_spaces && last_blank_pos) -+ { -+ fwrite (line_out, sizeof(char), last_blank_pos, stdout); -+ putchar ('\n'); -+ -+ offset_out = offset_out - last_blank_pos; -+ column = column - last_blank_column + ((is_cr_after_last_blank) -+ ? last_blank_increment : bs_following_last_blank_num); -+ memmove (line_out, line_out + last_blank_pos, offset_out); -+ CLEAR_FLAGS; -+ goto rescan; -+ } -+ -+ if (column + increment > width && column != 0) -+ { -+ fwrite (line_out, sizeof(char), offset_out, stdout); -+ START_NEW_LINE; -+ goto rescan; -+ } -+ -+ if (allocated_out < offset_out + mblength) -+ { -+ line_out = X2REALLOC (line_out, &allocated_out); -+ } -+ -+ memcpy (line_out + offset_out, bufpos, mblength); -+ offset_out += mblength; -+ column += increment; -+ -+ if (is_blank_seen && !convfail && wc == L'\r') -+ is_cr_after_last_blank = 1; -+ -+ if (is_bs_following_last_blank && !convfail && wc == L'\b') -+ ++bs_following_last_blank_num; -+ else -+ is_bs_following_last_blank = 0; -+ -+ if (break_spaces && !convfail && iswblank (wc)) -+ { -+ last_blank_pos = offset_out; -+ last_blank_column = column; -+ is_blank_seen = 1; -+ last_blank_increment = increment; -+ is_bs_following_last_blank = 1; -+ bs_following_last_blank_num = 0; -+ is_cr_after_last_blank = 0; -+ } -+ } -+ -+ *saved_errno = errno; - - if (offset_out) - fwrite (line_out, sizeof (char), (size_t) offset_out, stdout); - -+} -+#endif -+ -+/* Fold file FILENAME, or standard input if FILENAME is "-", -+ to stdout, with maximum line length WIDTH. -+ Return 0 if successful, 1 if an error occurs. */ -+ -+static bool -+fold_file (char const *filename, size_t width) -+{ -+ FILE *istream; -+ int saved_errno; -+ -+ if (STREQ (filename, "-")) -+ { -+ istream = stdin; -+ have_read_stdin = 1; -+ } -+ else -+ istream = fopen (filename, "r"); -+ -+ if (istream == NULL) -+ { -+ error (0, errno, "%s", quotef (filename)); -+ return 1; -+ } -+ -+ /* Define how ISTREAM is being folded. */ -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ fold_multibyte_text (istream, width, &saved_errno); -+ else -+#endif -+ fold_text (istream, width, &saved_errno); -+ - if (ferror (istream)) - { - error (0, saved_errno, "%s", quotef (filename)); -@@ -251,7 +498,8 @@ main (int argc, char **argv) - - atexit (close_stdout); - -- break_spaces = count_bytes = have_read_stdin = false; -+ operating_mode = column_mode; -+ break_spaces = have_read_stdin = false; - - while ((optc = getopt_long (argc, argv, shortopts, longopts, NULL)) != -1) - { -@@ -260,7 +508,15 @@ main (int argc, char **argv) - switch (optc) - { - case 'b': /* Count bytes rather than columns. */ -- count_bytes = true; -+ if (operating_mode != column_mode) -+ FATAL_ERROR (_("only one way of folding may be specified")); -+ operating_mode = byte_mode; -+ break; -+ -+ case 'c': -+ if (operating_mode != column_mode) -+ FATAL_ERROR (_("only one way of folding may be specified")); -+ operating_mode = character_mode; - break; - - case 's': /* Break at word boundaries. */ -diff -Naurp coreutils-8.25-orig/src/join.c coreutils-8.25/src/join.c ---- coreutils-8.25-orig/src/join.c 2016-01-13 05:08:59.000000000 -0600 -+++ coreutils-8.25/src/join.c 2016-02-08 19:07:10.303944625 -0600 -@@ -22,18 +22,32 @@ - #include - #include - -+/* Get mbstate_t, mbrtowc(), mbrtowc(), wcwidth(). */ -+#if HAVE_WCHAR_H -+# include -+#endif -+ -+/* Get iswblank(), towupper. */ -+#if HAVE_WCTYPE_H -+# include -+#endif -+ - #include "system.h" - #include "error.h" - #include "fadvise.h" - #include "hard-locale.h" - #include "linebuffer.h" --#include "memcasecmp.h" - #include "quote.h" - #include "stdio--.h" - #include "xmemcoll.h" - #include "xstrtol.h" - #include "argmatch.h" - -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif -+ - /* The official name of this program (e.g., no 'g' prefix). */ - #define PROGRAM_NAME "join" - -@@ -135,10 +149,12 @@ static struct outlist outlist_head; - /* Last element in 'outlist', where a new element can be added. */ - static struct outlist *outlist_end = &outlist_head; - --/* Tab character separating fields. If negative, fields are separated -- by any nonempty string of blanks, otherwise by exactly one -- tab character whose value (when cast to unsigned char) equals TAB. */ --static int tab = -1; -+/* Tab character separating fields. If NULL, fields are separated -+ by any nonempty string of blanks. */ -+static char *tab = NULL; -+ -+/* The number of bytes used for tab. */ -+static size_t tablen = 0; - - /* If nonzero, check that the input is correctly ordered. */ - static enum -@@ -275,13 +291,14 @@ xfields (struct line *line) - if (ptr == lim) - return; - -- if (0 <= tab && tab != '\n') -+ if (tab != NULL) - { -+ unsigned char t = tab[0]; - char *sep; -- for (; (sep = memchr (ptr, tab, lim - ptr)) != NULL; ptr = sep + 1) -+ for (; (sep = memchr (ptr, t, lim - ptr)) != NULL; ptr = sep + 1) - extract_field (line, ptr, sep - ptr); - } -- else if (tab < 0) -+ else - { - /* Skip leading blanks before the first field. */ - while (field_sep (*ptr)) -@@ -305,6 +322,147 @@ xfields (struct line *line) - extract_field (line, ptr, lim - ptr); - } - -+#if HAVE_MBRTOWC -+static void -+xfields_multibyte (struct line *line) -+{ -+ char *ptr = line->buf.buffer; -+ char const *lim = ptr + line->buf.length - 1; -+ wchar_t wc = 0; -+ size_t mblength = 1; -+ mbstate_t state, state_bak; -+ -+ memset (&state, 0, sizeof (mbstate_t)); -+ -+ if (ptr >= lim) -+ return; -+ -+ if (tab != NULL) -+ { -+ char *sep = ptr; -+ for (; ptr < lim; ptr = sep + mblength) -+ { -+ sep = ptr; -+ while (sep < lim) -+ { -+ state_bak = state; -+ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state); -+ -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ mblength = 1; -+ state = state_bak; -+ } -+ mblength = (mblength < 1) ? 1 : mblength; -+ -+ if (mblength == tablen && !memcmp (sep, tab, mblength)) -+ break; -+ else -+ { -+ sep += mblength; -+ continue; -+ } -+ } -+ -+ if (sep >= lim) -+ break; -+ -+ extract_field (line, ptr, sep - ptr); -+ } -+ } -+ else -+ { -+ /* Skip leading blanks before the first field. */ -+ while(ptr < lim) -+ { -+ state_bak = state; -+ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state); -+ -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ mblength = 1; -+ state = state_bak; -+ break; -+ } -+ mblength = (mblength < 1) ? 1 : mblength; -+ -+ if (!iswblank(wc) && wc != '\n') -+ break; -+ ptr += mblength; -+ } -+ -+ do -+ { -+ char *sep; -+ state_bak = state; -+ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state); -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ mblength = 1; -+ state = state_bak; -+ break; -+ } -+ mblength = (mblength < 1) ? 1 : mblength; -+ -+ sep = ptr + mblength; -+ while (sep < lim) -+ { -+ state_bak = state; -+ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state); -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ mblength = 1; -+ state = state_bak; -+ break; -+ } -+ mblength = (mblength < 1) ? 1 : mblength; -+ -+ if (iswblank (wc) || wc == '\n') -+ break; -+ -+ sep += mblength; -+ } -+ -+ extract_field (line, ptr, sep - ptr); -+ if (sep >= lim) -+ return; -+ -+ state_bak = state; -+ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state); -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ mblength = 1; -+ state = state_bak; -+ break; -+ } -+ mblength = (mblength < 1) ? 1 : mblength; -+ -+ ptr = sep + mblength; -+ while (ptr < lim) -+ { -+ state_bak = state; -+ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state); -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ mblength = 1; -+ state = state_bak; -+ break; -+ } -+ mblength = (mblength < 1) ? 1 : mblength; -+ -+ if (!iswblank (wc) && wc != '\n') -+ break; -+ -+ ptr += mblength; -+ } -+ } -+ while (ptr < lim); -+ } -+ -+ extract_field (line, ptr, lim - ptr); -+} -+#endif -+ - static void - freeline (struct line *line) - { -@@ -326,56 +484,133 @@ keycmp (struct line const *line1, struct - size_t jf_1, size_t jf_2) - { - /* Start of field to compare in each file. */ -- char *beg1; -- char *beg2; -- -- size_t len1; -- size_t len2; /* Length of fields to compare. */ -+ char *beg[2]; -+ char *copy[2]; -+ size_t len[2]; /* Length of fields to compare. */ - int diff; -+ int i, j; -+ int mallocd = 0; - - if (jf_1 < line1->nfields) - { -- beg1 = line1->fields[jf_1].beg; -- len1 = line1->fields[jf_1].len; -+ beg[0] = line1->fields[jf_1].beg; -+ len[0] = line1->fields[jf_1].len; - } - else - { -- beg1 = NULL; -- len1 = 0; -+ beg[0] = NULL; -+ len[0] = 0; - } - - if (jf_2 < line2->nfields) - { -- beg2 = line2->fields[jf_2].beg; -- len2 = line2->fields[jf_2].len; -+ beg[1] = line2->fields[jf_2].beg; -+ len[1] = line2->fields[jf_2].len; - } - else - { -- beg2 = NULL; -- len2 = 0; -+ beg[1] = NULL; -+ len[1] = 0; - } - -- if (len1 == 0) -- return len2 == 0 ? 0 : -1; -- if (len2 == 0) -+ if (len[0] == 0) -+ return len[1] == 0 ? 0 : -1; -+ if (len[1] == 0) - return 1; - - if (ignore_case) - { -- /* FIXME: ignore_case does not work with NLS (in particular, -- with multibyte chars). */ -- diff = memcasecmp (beg1, beg2, MIN (len1, len2)); -+#ifdef HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ size_t mblength; -+ wchar_t wc, uwc; -+ mbstate_t state, state_bak; -+ -+ memset (&state, '\0', sizeof (mbstate_t)); -+ -+ for (i = 0; i < 2; i++) -+ { -+ mallocd = 1; -+ copy[i] = xmalloc (len[i] + 1); -+ memset (copy[i], '\0',len[i] + 1); -+ -+ for (j = 0; j < MIN (len[0], len[1]);) -+ { -+ state_bak = state; -+ mblength = mbrtowc (&wc, beg[i] + j, len[i] - j, &state); -+ -+ switch (mblength) -+ { -+ case (size_t) -1: -+ case (size_t) -2: -+ state = state_bak; -+ /* Fall through */ -+ case 0: -+ mblength = 1; -+ break; -+ -+ default: -+ uwc = towupper (wc); -+ -+ if (uwc != wc) -+ { -+ mbstate_t state_wc; -+ size_t mblen; -+ -+ memset (&state_wc, '\0', sizeof (mbstate_t)); -+ mblen = wcrtomb (copy[i] + j, uwc, &state_wc); -+ assert (mblen != (size_t)-1); -+ } -+ else -+ memcpy (copy[i] + j, beg[i] + j, mblength); -+ } -+ j += mblength; -+ } -+ copy[i][j] = '\0'; -+ } -+ } -+ else -+#endif -+ { -+ for (i = 0; i < 2; i++) -+ { -+ mallocd = 1; -+ copy[i] = xmalloc (len[i] + 1); -+ -+ for (j = 0; j < MIN (len[0], len[1]); j++) -+ copy[i][j] = toupper (beg[i][j]); -+ -+ copy[i][j] = '\0'; -+ } -+ } - } - else - { -- if (hard_LC_COLLATE) -- return xmemcoll (beg1, len1, beg2, len2); -- diff = memcmp (beg1, beg2, MIN (len1, len2)); -+ copy[0] = beg[0]; -+ copy[1] = beg[1]; -+ } -+ -+ if (hard_LC_COLLATE) -+ { -+ diff = xmemcoll ((char *) copy[0], len[0], (char *) copy[1], len[1]); -+ -+ if (mallocd) -+ for (i = 0; i < 2; i++) -+ free (copy[i]); -+ -+ return diff; - } -+ diff = memcmp (copy[0], copy[1], MIN (len[0], len[1])); -+ -+ if (mallocd) -+ for (i = 0; i < 2; i++) -+ free (copy[i]); -+ - - if (diff) - return diff; -- return len1 < len2 ? -1 : len1 != len2; -+ return len[0] - len[1]; - } - - /* Check that successive input lines PREV and CURRENT from input file -@@ -467,6 +702,11 @@ get_line (FILE *fp, struct line **linep, - } - ++line_no[which - 1]; - -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ xfields_multibyte (line); -+ else -+#endif - xfields (line); - - if (prevline[which - 1]) -@@ -566,21 +806,28 @@ prfield (size_t n, struct line const *li - - /* Output all the fields in line, other than the join field. */ - -+#define PUT_TAB_CHAR \ -+ do \ -+ { \ -+ (tab != NULL) ? \ -+ fwrite(tab, sizeof(char), tablen, stdout) : putchar (' '); \ -+ } \ -+ while (0) -+ - static void - prfields (struct line const *line, size_t join_field, size_t autocount) - { - size_t i; - size_t nfields = autoformat ? autocount : line->nfields; -- char output_separator = tab < 0 ? ' ' : tab; - - for (i = 0; i < join_field && i < nfields; ++i) - { -- putchar (output_separator); -+ PUT_TAB_CHAR; - prfield (i, line); - } - for (i = join_field + 1; i < nfields; ++i) - { -- putchar (output_separator); -+ PUT_TAB_CHAR; - prfield (i, line); - } - } -@@ -591,7 +838,6 @@ static void - prjoin (struct line const *line1, struct line const *line2) - { - const struct outlist *outlist; -- char output_separator = tab < 0 ? ' ' : tab; - size_t field; - struct line const *line; - -@@ -625,7 +871,7 @@ prjoin (struct line const *line1, struct - o = o->next; - if (o == NULL) - break; -- putchar (output_separator); -+ PUT_TAB_CHAR; - } - putchar (eolchar); - } -@@ -1103,21 +1349,46 @@ main (int argc, char **argv) - - case 't': - { -- unsigned char newtab = optarg[0]; -+ char *newtab = NULL; -+ size_t newtablen; -+ newtab = xstrdup (optarg); -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ mbstate_t state; -+ -+ memset (&state, 0, sizeof (mbstate_t)); -+ newtablen = mbrtowc (NULL, newtab, -+ strnlen (newtab, MB_LEN_MAX), -+ &state); -+ if (newtablen == (size_t) 0 -+ || newtablen == (size_t) -1 -+ || newtablen == (size_t) -2) -+ newtablen = 1; -+ } -+ else -+#endif -+ newtablen = 1; - if (! newtab) -- newtab = '\n'; /* '' => process the whole line. */ -+ { -+ newtab = (char*)"\n"; /* '' => process the whole line. */ -+ } - else if (optarg[1]) - { -- if (STREQ (optarg, "\\0")) -- newtab = '\0'; -- else -- error (EXIT_FAILURE, 0, _("multi-character tab %s"), -- quote (optarg)); -+ if (newtablen == 1 && newtab[1]) -+ { -+ if (STREQ (newtab, "\\0")) -+ newtab[0] = '\0'; -+ } -+ } -+ if (tab != NULL && strcmp (tab, newtab)) -+ { -+ free (newtab); -+ error (EXIT_FAILURE, 0, _("incompatible tabs")); - } -- if (0 <= tab && tab != newtab) -- error (EXIT_FAILURE, 0, _("incompatible tabs")); - tab = newtab; -- } -+ tablen = newtablen; -+ } - break; - - case 'z': -diff -Naurp coreutils-8.25-orig/src/pr.c coreutils-8.25/src/pr.c ---- coreutils-8.25-orig/src/pr.c 2016-01-01 07:48:50.000000000 -0600 -+++ coreutils-8.25/src/pr.c 2016-02-08 19:07:10.306944635 -0600 -@@ -311,6 +311,24 @@ - - #include - #include -+ -+/* Get MB_LEN_MAX. */ -+#include -+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ -+#if !defined MB_LEN_MAX || MB_LEN_MAX == 1 -+# define MB_LEN_MAX 16 -+#endif -+ -+/* Get MB_CUR_MAX. */ -+#include -+ -+/* Solaris 2.5 has a bug: must be included before . */ -+/* Get mbstate_t, mbrtowc(), wcwidth(). */ -+#if HAVE_WCHAR_H -+# include -+#endif -+ - #include "system.h" - #include "error.h" - #include "fadvise.h" -@@ -323,6 +341,18 @@ - #include "xstrtol.h" - #include "xdectoint.h" - -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif -+ -+#ifndef HAVE_DECL_WCWIDTH -+"this configure-time declaration test was not run" -+#endif -+#if !HAVE_DECL_WCWIDTH -+extern int wcwidth (); -+#endif -+ - /* The official name of this program (e.g., no 'g' prefix). */ - #define PROGRAM_NAME "pr" - -@@ -415,7 +445,20 @@ struct COLUMN - - typedef struct COLUMN COLUMN; - --static int char_to_clump (char c); -+/* Funtion pointers to switch functions for single byte locale or for -+ multibyte locale. If multibyte functions do not exist in your sysytem, -+ these pointers always point the function for single byte locale. */ -+static void (*print_char) (char c); -+static int (*char_to_clump) (char c); -+ -+/* Functions for single byte locale. */ -+static void print_char_single (char c); -+static int char_to_clump_single (char c); -+ -+/* Functions for multibyte locale. */ -+static void print_char_multi (char c); -+static int char_to_clump_multi (char c); -+ - static bool read_line (COLUMN *p); - static bool print_page (void); - static bool print_stored (COLUMN *p); -@@ -427,6 +470,7 @@ static void add_line_number (COLUMN *p); - static void getoptnum (const char *n_str, int min, int *num, - const char *errfmt); - static void getoptarg (char *arg, char switch_char, char *character, -+ int *character_length, int *character_width, - int *number); - static void print_files (int number_of_files, char **av); - static void init_parameters (int number_of_files); -@@ -440,7 +484,6 @@ static void store_char (char c); - static void pad_down (unsigned int lines); - static void read_rest_of_line (COLUMN *p); - static void skip_read (COLUMN *p, int column_number); --static void print_char (char c); - static void cleanup (void); - static void print_sep_string (void); - static void separator_string (const char *optarg_S); -@@ -452,7 +495,7 @@ static COLUMN *column_vector; - we store the leftmost columns contiguously in buff. - To print a line from buff, get the index of the first character - from line_vector[i], and print up to line_vector[i + 1]. */ --static char *buff; -+static unsigned char *buff; - - /* Index of the position in buff where the next character - will be stored. */ -@@ -556,7 +599,7 @@ static int chars_per_column; - static bool untabify_input = false; - - /* (-e) The input tab character. */ --static char input_tab_char = '\t'; -+static char input_tab_char[MB_LEN_MAX] = "\t"; - - /* (-e) Tabstops are at chars_per_tab, 2*chars_per_tab, 3*chars_per_tab, ... - where the leftmost column is 1. */ -@@ -566,7 +609,10 @@ static int chars_per_input_tab = 8; - static bool tabify_output = false; - - /* (-i) The output tab character. */ --static char output_tab_char = '\t'; -+static char output_tab_char[MB_LEN_MAX] = "\t"; -+ -+/* (-i) The byte length of output tab character. */ -+static int output_tab_char_length = 1; - - /* (-i) The width of the output tab. */ - static int chars_per_output_tab = 8; -@@ -636,7 +682,13 @@ static int line_number; - static bool numbered_lines = false; - - /* (-n) Character which follows each line number. */ --static char number_separator = '\t'; -+static char number_separator[MB_LEN_MAX] = "\t"; -+ -+/* (-n) The byte length of the character which follows each line number. */ -+static int number_separator_length = 1; -+ -+/* (-n) The character width of the character which follows each line number. */ -+static int number_separator_width = 0; - - /* (-n) line counting starts with 1st line of input file (not with 1st - line of 1st page printed). */ -@@ -689,6 +741,7 @@ static bool use_col_separator = false; - -a|COLUMN|-m is a 'space' and with the -J option a 'tab'. */ - static char *col_sep_string = (char *) ""; - static int col_sep_length = 0; -+static int col_sep_width = 0; - static char *column_separator = (char *) " "; - static char *line_separator = (char *) "\t"; - -@@ -839,6 +892,13 @@ separator_string (const char *optarg_S) - col_sep_length = (int) strlen (optarg_S); - col_sep_string = xmalloc (col_sep_length + 1); - strcpy (col_sep_string, optarg_S); -+ -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ col_sep_width = mbswidth (col_sep_string, 0); -+ else -+#endif -+ col_sep_width = col_sep_length; - } - - int -@@ -863,6 +923,21 @@ main (int argc, char **argv) - - atexit (close_stdout); - -+/* Define which functions are used, the ones for single byte locale or the ones -+ for multibyte locale. */ -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ print_char = print_char_multi; -+ char_to_clump = char_to_clump_multi; -+ } -+ else -+#endif -+ { -+ print_char = print_char_single; -+ char_to_clump = char_to_clump_single; -+ } -+ - n_files = 0; - file_names = (argc > 1 - ? xmalloc ((argc - 1) * sizeof (char *)) -@@ -939,8 +1014,12 @@ main (int argc, char **argv) - break; - case 'e': - if (optarg) -- getoptarg (optarg, 'e', &input_tab_char, -- &chars_per_input_tab); -+ { -+ int dummy_length, dummy_width; -+ -+ getoptarg (optarg, 'e', input_tab_char, &dummy_length, -+ &dummy_width, &chars_per_input_tab); -+ } - /* Could check tab width > 0. */ - untabify_input = true; - break; -@@ -953,8 +1032,12 @@ main (int argc, char **argv) - break; - case 'i': - if (optarg) -- getoptarg (optarg, 'i', &output_tab_char, -- &chars_per_output_tab); -+ { -+ int dummy_width; -+ -+ getoptarg (optarg, 'i', output_tab_char, &output_tab_char_length, -+ &dummy_width, &chars_per_output_tab); -+ } - /* Could check tab width > 0. */ - tabify_output = true; - break; -@@ -972,8 +1055,8 @@ main (int argc, char **argv) - case 'n': - numbered_lines = true; - if (optarg) -- getoptarg (optarg, 'n', &number_separator, -- &chars_per_number); -+ getoptarg (optarg, 'n', number_separator, &number_separator_length, -+ &number_separator_width, &chars_per_number); - break; - case 'N': - skip_count = false; -@@ -997,7 +1080,7 @@ main (int argc, char **argv) - old_s = false; - /* Reset an additional input of -s, -S dominates -s */ - col_sep_string = bad_cast (""); -- col_sep_length = 0; -+ col_sep_length = col_sep_width = 0; - use_col_separator = true; - if (optarg) - separator_string (optarg); -@@ -1152,10 +1235,45 @@ getoptnum (const char *n_str, int min, i - a number. */ - - static void --getoptarg (char *arg, char switch_char, char *character, int *number) -+getoptarg (char *arg, char switch_char, char *character, int *character_length, -+ int *character_width, int *number) - { - if (!ISDIGIT (*arg)) -- *character = *arg++; -+ { -+#ifdef HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) /* for multibyte locale. */ -+ { -+ wchar_t wc; -+ size_t mblength; -+ int width; -+ mbstate_t state = {'\0'}; -+ -+ mblength = mbrtowc (&wc, arg, strnlen(arg, MB_LEN_MAX), &state); -+ -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ *character_length = 1; -+ *character_width = 1; -+ } -+ else -+ { -+ *character_length = (mblength < 1) ? 1 : mblength; -+ width = wcwidth (wc); -+ *character_width = (width < 0) ? 0 : width; -+ } -+ -+ strncpy (character, arg, *character_length); -+ arg += *character_length; -+ } -+ else /* for single byte locale. */ -+#endif -+ { -+ *character = *arg++; -+ *character_length = 1; -+ *character_width = 1; -+ } -+ } -+ - if (*arg) - { - long int tmp_long; -@@ -1177,6 +1295,11 @@ static void - init_parameters (int number_of_files) - { - int chars_used_by_number = 0; -+ int mb_len = 1; -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ mb_len = MB_LEN_MAX; -+#endif - - lines_per_body = lines_per_page - lines_per_header - lines_per_footer; - if (lines_per_body <= 0) -@@ -1214,7 +1337,7 @@ init_parameters (int number_of_files) - else - col_sep_string = column_separator; - -- col_sep_length = 1; -+ col_sep_length = col_sep_width = 1; - use_col_separator = true; - } - /* It's rather pointless to define a TAB separator with column -@@ -1244,11 +1367,11 @@ init_parameters (int number_of_files) - + TAB_WIDTH (chars_per_input_tab, chars_per_number); */ - - /* Estimate chars_per_text without any margin and keep it constant. */ -- if (number_separator == '\t') -+ if (number_separator[0] == '\t') - number_width = (chars_per_number - + TAB_WIDTH (chars_per_default_tab, chars_per_number)); - else -- number_width = chars_per_number + 1; -+ number_width = chars_per_number + number_separator_width; - - /* The number is part of the column width unless we are - printing files in parallel. */ -@@ -1257,7 +1380,7 @@ init_parameters (int number_of_files) - } - - chars_per_column = (chars_per_line - chars_used_by_number -- - (columns - 1) * col_sep_length) / columns; -+ - (columns - 1) * col_sep_width) / columns; - - if (chars_per_column < 1) - error (EXIT_FAILURE, 0, _("page width too narrow")); -@@ -1275,7 +1398,7 @@ init_parameters (int number_of_files) - We've to use 8 as the lower limit, if we use chars_per_default_tab = 8 - to expand a tab which is not an input_tab-char. */ - free (clump_buff); -- clump_buff = xmalloc (MAX (8, chars_per_input_tab)); -+ clump_buff = xmalloc (mb_len * MAX (8, chars_per_input_tab)); - } - - /* Open the necessary files, -@@ -1383,7 +1506,7 @@ init_funcs (void) - - /* Enlarge p->start_position of first column to use the same form of - padding_not_printed with all columns. */ -- h = h + col_sep_length; -+ h = h + col_sep_width; - - /* This loop takes care of all but the rightmost column. */ - -@@ -1417,7 +1540,7 @@ init_funcs (void) - } - else - { -- h = h_next + col_sep_length; -+ h = h_next + col_sep_width; - h_next = h + chars_per_column; - } - } -@@ -1708,9 +1831,9 @@ static void - align_column (COLUMN *p) - { - padding_not_printed = p->start_position; -- if (padding_not_printed - col_sep_length > 0) -+ if (padding_not_printed - col_sep_width > 0) - { -- pad_across_to (padding_not_printed - col_sep_length); -+ pad_across_to (padding_not_printed - col_sep_width); - padding_not_printed = ANYWHERE; - } - -@@ -1981,13 +2104,13 @@ store_char (char c) - /* May be too generous. */ - buff = X2REALLOC (buff, &buff_allocated); - } -- buff[buff_current++] = c; -+ buff[buff_current++] = (unsigned char) c; - } - - static void - add_line_number (COLUMN *p) - { -- int i; -+ int i, j; - char *s; - int num_width; - -@@ -2004,22 +2127,24 @@ add_line_number (COLUMN *p) - /* Tabification is assumed for multiple columns, also for n-separators, - but 'default n-separator = TAB' hasn't been given priority over - equal column_width also specified by POSIX. */ -- if (number_separator == '\t') -+ if (number_separator[0] == '\t') - { - i = number_width - chars_per_number; - while (i-- > 0) - (p->char_func) (' '); - } - else -- (p->char_func) (number_separator); -+ for (j = 0; j < number_separator_length; j++) -+ (p->char_func) (number_separator[j]); - } - else - /* To comply with POSIX, we avoid any expansion of default TAB - separator with a single column output. No column_width requirement - has to be considered. */ - { -- (p->char_func) (number_separator); -- if (number_separator == '\t') -+ for (j = 0; j < number_separator_length; j++) -+ (p->char_func) (number_separator[j]); -+ if (number_separator[0] == '\t') - output_position = POS_AFTER_TAB (chars_per_output_tab, - output_position); - } -@@ -2180,7 +2305,7 @@ print_white_space (void) - while (goal - h_old > 1 - && (h_new = POS_AFTER_TAB (chars_per_output_tab, h_old)) <= goal) - { -- putchar (output_tab_char); -+ fwrite (output_tab_char, sizeof(char), output_tab_char_length, stdout); - h_old = h_new; - } - while (++h_old <= goal) -@@ -2200,6 +2325,7 @@ print_sep_string (void) - { - char *s; - int l = col_sep_length; -+ int not_space_flag; - - s = col_sep_string; - -@@ -2213,6 +2339,7 @@ print_sep_string (void) - { - for (; separators_not_printed > 0; --separators_not_printed) - { -+ not_space_flag = 0; - while (l-- > 0) - { - /* 3 types of sep_strings: spaces only, spaces and chars, -@@ -2226,12 +2353,15 @@ print_sep_string (void) - } - else - { -+ not_space_flag = 1; - if (spaces_not_printed > 0) - print_white_space (); - putchar (*s++); -- ++output_position; - } - } -+ if (not_space_flag) -+ output_position += col_sep_width; -+ - /* sep_string ends with some spaces */ - if (spaces_not_printed > 0) - print_white_space (); -@@ -2259,7 +2389,7 @@ print_clump (COLUMN *p, int n, char *clu - required number of tabs and spaces. */ - - static void --print_char (char c) -+print_char_single (char c) - { - if (tabify_output) - { -@@ -2283,6 +2413,74 @@ print_char (char c) - putchar (c); - } - -+#ifdef HAVE_MBRTOWC -+static void -+print_char_multi (char c) -+{ -+ static size_t mbc_pos = 0; -+ static char mbc[MB_LEN_MAX] = {'\0'}; -+ static mbstate_t state = {'\0'}; -+ mbstate_t state_bak; -+ wchar_t wc; -+ size_t mblength; -+ int width; -+ -+ if (tabify_output) -+ { -+ state_bak = state; -+ mbc[mbc_pos++] = c; -+ mblength = mbrtowc (&wc, mbc, mbc_pos, &state); -+ -+ while (mbc_pos > 0) -+ { -+ switch (mblength) -+ { -+ case (size_t)-2: -+ state = state_bak; -+ return; -+ -+ case (size_t)-1: -+ state = state_bak; -+ ++output_position; -+ putchar (mbc[0]); -+ memmove (mbc, mbc + 1, MB_CUR_MAX - 1); -+ --mbc_pos; -+ break; -+ -+ case 0: -+ mblength = 1; -+ -+ default: -+ if (wc == L' ') -+ { -+ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength); -+ --mbc_pos; -+ ++spaces_not_printed; -+ return; -+ } -+ else if (spaces_not_printed > 0) -+ print_white_space (); -+ -+ /* Nonprintables are assumed to have width 0, except L'\b'. */ -+ if ((width = wcwidth (wc)) < 1) -+ { -+ if (wc == L'\b') -+ --output_position; -+ } -+ else -+ output_position += width; -+ -+ fwrite (mbc, sizeof(char), mblength, stdout); -+ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength); -+ mbc_pos -= mblength; -+ } -+ } -+ return; -+ } -+ putchar (c); -+} -+#endif -+ - /* Skip to page PAGE before printing. - PAGE may be larger than total number of pages. */ - -@@ -2462,9 +2660,9 @@ read_line (COLUMN *p) - align_empty_cols = false; - } - -- if (padding_not_printed - col_sep_length > 0) -+ if (padding_not_printed - col_sep_width > 0) - { -- pad_across_to (padding_not_printed - col_sep_length); -+ pad_across_to (padding_not_printed - col_sep_width); - padding_not_printed = ANYWHERE; - } - -@@ -2534,7 +2732,7 @@ print_stored (COLUMN *p) - int i; - - int line = p->current_line++; -- char *first = &buff[line_vector[line]]; -+ unsigned char *first = &buff[line_vector[line]]; - /* FIXME - UMR: Uninitialized memory read: - * This is occurring while in: -@@ -2546,7 +2744,7 @@ print_stored (COLUMN *p) - xmalloc [xmalloc.c:94] - init_store_cols [pr.c:1648] - */ -- char *last = &buff[line_vector[line + 1]]; -+ unsigned char *last = &buff[line_vector[line + 1]]; - - pad_vertically = true; - -@@ -2565,9 +2763,9 @@ print_stored (COLUMN *p) - } - } - -- if (padding_not_printed - col_sep_length > 0) -+ if (padding_not_printed - col_sep_width > 0) - { -- pad_across_to (padding_not_printed - col_sep_length); -+ pad_across_to (padding_not_printed - col_sep_width); - padding_not_printed = ANYWHERE; - } - -@@ -2580,8 +2778,8 @@ print_stored (COLUMN *p) - if (spaces_not_printed == 0) - { - output_position = p->start_position + end_vector[line]; -- if (p->start_position - col_sep_length == chars_per_margin) -- output_position -= col_sep_length; -+ if (p->start_position - col_sep_width == chars_per_margin) -+ output_position -= col_sep_width; - } - - return true; -@@ -2600,7 +2798,7 @@ print_stored (COLUMN *p) - number of characters is 1.) */ - - static int --char_to_clump (char c) -+char_to_clump_single (char c) - { - unsigned char uc = c; - char *s = clump_buff; -@@ -2610,10 +2808,10 @@ char_to_clump (char c) - int chars; - int chars_per_c = 8; - -- if (c == input_tab_char) -+ if (c == input_tab_char[0]) - chars_per_c = chars_per_input_tab; - -- if (c == input_tab_char || c == '\t') -+ if (c == input_tab_char[0] || c == '\t') - { - width = TAB_WIDTH (chars_per_c, input_position); - -@@ -2694,6 +2892,164 @@ char_to_clump (char c) - return chars; - } - -+#ifdef HAVE_MBRTOWC -+static int -+char_to_clump_multi (char c) -+{ -+ static size_t mbc_pos = 0; -+ static char mbc[MB_LEN_MAX] = {'\0'}; -+ static mbstate_t state = {'\0'}; -+ mbstate_t state_bak; -+ wchar_t wc; -+ size_t mblength; -+ int wc_width; -+ register char *s = clump_buff; -+ register int i, j; -+ char esc_buff[4]; -+ int width; -+ int chars; -+ int chars_per_c = 8; -+ -+ state_bak = state; -+ mbc[mbc_pos++] = c; -+ mblength = mbrtowc (&wc, mbc, mbc_pos, &state); -+ -+ width = 0; -+ chars = 0; -+ while (mbc_pos > 0) -+ { -+ switch (mblength) -+ { -+ case (size_t)-2: -+ state = state_bak; -+ return 0; -+ -+ case (size_t)-1: -+ state = state_bak; -+ mblength = 1; -+ -+ if (use_esc_sequence || use_cntrl_prefix) -+ { -+ width = +4; -+ chars = +4; -+ *s++ = '\\'; -+ sprintf (esc_buff, "%03o", (unsigned char) mbc[0]); -+ for (i = 0; i <= 2; ++i) -+ *s++ = (int) esc_buff[i]; -+ } -+ else -+ { -+ width += 1; -+ chars += 1; -+ *s++ = mbc[0]; -+ } -+ break; -+ -+ case 0: -+ mblength = 1; -+ /* Fall through */ -+ -+ default: -+ if (memcmp (mbc, input_tab_char, mblength) == 0) -+ chars_per_c = chars_per_input_tab; -+ -+ if (memcmp (mbc, input_tab_char, mblength) == 0 || c == '\t') -+ { -+ int width_inc; -+ -+ width_inc = TAB_WIDTH (chars_per_c, input_position); -+ width += width_inc; -+ -+ if (untabify_input) -+ { -+ for (i = width_inc; i; --i) -+ *s++ = ' '; -+ chars += width_inc; -+ } -+ else -+ { -+ for (i = 0; i < mblength; i++) -+ *s++ = mbc[i]; -+ chars += mblength; -+ } -+ } -+ else if ((wc_width = wcwidth (wc)) < 1) -+ { -+ if (use_esc_sequence) -+ { -+ for (i = 0; i < mblength; i++) -+ { -+ width += 4; -+ chars += 4; -+ *s++ = '\\'; -+ sprintf (esc_buff, "%03o", (unsigned char) mbc[i]); -+ for (j = 0; j <= 2; ++j) -+ *s++ = (int) esc_buff[j]; -+ } -+ } -+ else if (use_cntrl_prefix) -+ { -+ if (wc < 0200) -+ { -+ width += 2; -+ chars += 2; -+ *s++ = '^'; -+ *s++ = wc ^ 0100; -+ } -+ else -+ { -+ for (i = 0; i < mblength; i++) -+ { -+ width += 4; -+ chars += 4; -+ *s++ = '\\'; -+ sprintf (esc_buff, "%03o", (unsigned char) mbc[i]); -+ for (j = 0; j <= 2; ++j) -+ *s++ = (int) esc_buff[j]; -+ } -+ } -+ } -+ else if (wc == L'\b') -+ { -+ width += -1; -+ chars += 1; -+ *s++ = c; -+ } -+ else -+ { -+ width += 0; -+ chars += mblength; -+ for (i = 0; i < mblength; i++) -+ *s++ = mbc[i]; -+ } -+ } -+ else -+ { -+ width += wc_width; -+ chars += mblength; -+ for (i = 0; i < mblength; i++) -+ *s++ = mbc[i]; -+ } -+ } -+ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength); -+ mbc_pos -= mblength; -+ } -+ -+ /* Too many backspaces must put us in position 0 -- never negative. */ -+ if (width < 0 && input_position == 0) -+ { -+ chars = 0; -+ input_position = 0; -+ } -+ else if (width < 0 && input_position <= -width) -+ input_position = 0; -+ else -+ input_position += width; -+ -+ return chars; -+} -+#endif -+ - /* We've just printed some files and need to clean up things before - looking for more options and printing the next batch of files. - -diff -Naurp coreutils-8.25-orig/src/sort.c coreutils-8.25/src/sort.c ---- coreutils-8.25-orig/src/sort.c 2016-01-16 13:09:33.000000000 -0600 -+++ coreutils-8.25/src/sort.c 2016-02-08 19:07:10.310944648 -0600 -@@ -29,6 +29,14 @@ - #include - #include - #include -+#if HAVE_WCHAR_H -+# include -+#endif -+/* Get isw* functions. */ -+#if HAVE_WCTYPE_H -+# include -+#endif -+ - #include "system.h" - #include "argmatch.h" - #include "error.h" -@@ -163,14 +171,39 @@ static int decimal_point; - /* Thousands separator; if -1, then there isn't one. */ - static int thousands_sep; - -+/* True if -f is specified. */ -+static bool folding; -+ - /* Nonzero if the corresponding locales are hard. */ - static bool hard_LC_COLLATE; --#if HAVE_NL_LANGINFO -+#if HAVE_LANGINFO_CODESET - static bool hard_LC_TIME; - #endif - - #define NONZERO(x) ((x) != 0) - -+/* get a multibyte character's byte length. */ -+#define GET_BYTELEN_OF_CHAR(LIM, PTR, MBLENGTH, STATE) \ -+ do \ -+ { \ -+ wchar_t wc; \ -+ mbstate_t state_bak; \ -+ \ -+ state_bak = STATE; \ -+ mblength = mbrtowc (&wc, PTR, LIM - PTR, &STATE); \ -+ \ -+ switch (MBLENGTH) \ -+ { \ -+ case (size_t)-1: \ -+ case (size_t)-2: \ -+ STATE = state_bak; \ -+ /* Fall through. */ \ -+ case 0: \ -+ MBLENGTH = 1; \ -+ } \ -+ } \ -+ while (0) -+ - /* The kind of blanks for '-b' to skip in various options. */ - enum blanktype { bl_start, bl_end, bl_both }; - -@@ -344,13 +377,11 @@ static bool reverse; - they were read if all keys compare equal. */ - static bool stable; - --/* If TAB has this value, blanks separate fields. */ --enum { TAB_DEFAULT = CHAR_MAX + 1 }; -- --/* Tab character separating fields. If TAB_DEFAULT, then fields are -+/* Tab character separating fields. If tab_length is 0, then fields are - separated by the empty string between a non-blank character and a blank - character. */ --static int tab = TAB_DEFAULT; -+static char tab[MB_LEN_MAX + 1]; -+static size_t tab_length = 0; - - /* Flag to remove consecutive duplicate lines from the output. - Only the last of a sequence of equal lines will be output. */ -@@ -810,6 +841,46 @@ reap_all (void) - reap (-1); - } - -+/* Function pointers. */ -+static void -+(*inittables) (void); -+static char * -+(*begfield) (const struct line*, const struct keyfield *); -+static char * -+(*limfield) (const struct line*, const struct keyfield *); -+static void -+(*skipblanks) (char **ptr, char *lim); -+static int -+(*getmonth) (char const *, size_t, char **); -+static int -+(*keycompare) (const struct line *, const struct line *); -+static int -+(*numcompare) (const char *, const char *); -+ -+/* Test for white space multibyte character. -+ Set LENGTH the byte length of investigated multibyte character. */ -+#if HAVE_MBRTOWC -+static int -+ismbblank (const char *str, size_t len, size_t *length) -+{ -+ size_t mblength; -+ wchar_t wc; -+ mbstate_t state; -+ -+ memset (&state, '\0', sizeof(mbstate_t)); -+ mblength = mbrtowc (&wc, str, len, &state); -+ -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ *length = 1; -+ return 0; -+ } -+ -+ *length = (mblength < 1) ? 1 : mblength; -+ return iswblank (wc) || wc == '\n'; -+} -+#endif -+ - /* Clean up any remaining temporary files. */ - - static void -@@ -1254,7 +1325,7 @@ zaptemp (char const *name) - free (node); - } - --#if HAVE_NL_LANGINFO -+#if HAVE_LANGINFO_CODESET - - static int - struct_month_cmp (void const *m1, void const *m2) -@@ -1269,7 +1340,7 @@ struct_month_cmp (void const *m1, void c - /* Initialize the character class tables. */ - - static void --inittables (void) -+inittables_uni (void) - { - size_t i; - -@@ -1281,7 +1352,7 @@ inittables (void) - fold_toupper[i] = toupper (i); - } - --#if HAVE_NL_LANGINFO -+#if HAVE_LANGINFO_CODESET - /* If we're not in the "C" locale, read different names for months. */ - if (hard_LC_TIME) - { -@@ -1363,6 +1434,84 @@ specify_nmerge (int oi, char c, char con - xstrtol_fatal (e, oi, c, long_options, s); - } - -+#if HAVE_MBRTOWC -+static void -+inittables_mb (void) -+{ -+ int i, j, k, l; -+ char *name, *s, *lc_time, *lc_ctype; -+ size_t s_len, mblength; -+ char mbc[MB_LEN_MAX]; -+ wchar_t wc, pwc; -+ mbstate_t state_mb, state_wc; -+ -+ lc_time = setlocale (LC_TIME, ""); -+ if (lc_time) -+ lc_time = xstrdup (lc_time); -+ -+ lc_ctype = setlocale (LC_CTYPE, ""); -+ if (lc_ctype) -+ lc_ctype = xstrdup (lc_ctype); -+ -+ if (lc_time && lc_ctype) -+ /* temporarily set LC_CTYPE to match LC_TIME, so that we can convert -+ * the names of months to upper case */ -+ setlocale (LC_CTYPE, lc_time); -+ -+ for (i = 0; i < MONTHS_PER_YEAR; i++) -+ { -+ s = (char *) nl_langinfo (ABMON_1 + i); -+ s_len = strlen (s); -+ monthtab[i].name = name = (char *) xmalloc (s_len + 1); -+ monthtab[i].val = i + 1; -+ -+ memset (&state_mb, '\0', sizeof (mbstate_t)); -+ memset (&state_wc, '\0', sizeof (mbstate_t)); -+ -+ for (j = 0; j < s_len;) -+ { -+ if (!ismbblank (s + j, s_len - j, &mblength)) -+ break; -+ j += mblength; -+ } -+ -+ for (k = 0; j < s_len;) -+ { -+ mblength = mbrtowc (&wc, (s + j), (s_len - j), &state_mb); -+ assert (mblength != (size_t)-1 && mblength != (size_t)-2); -+ if (mblength == 0) -+ break; -+ -+ pwc = towupper (wc); -+ if (pwc == wc) -+ { -+ memcpy (mbc, s + j, mblength); -+ j += mblength; -+ } -+ else -+ { -+ j += mblength; -+ mblength = wcrtomb (mbc, pwc, &state_wc); -+ assert (mblength != (size_t)0 && mblength != (size_t)-1); -+ } -+ -+ for (l = 0; l < mblength; l++) -+ name[k++] = mbc[l]; -+ } -+ name[k] = '\0'; -+ } -+ qsort ((void *) monthtab, MONTHS_PER_YEAR, -+ sizeof (struct month), struct_month_cmp); -+ -+ if (lc_time && lc_ctype) -+ /* restore the original locales */ -+ setlocale (LC_CTYPE, lc_ctype); -+ -+ free (lc_ctype); -+ free (lc_time); -+} -+#endif -+ - /* Specify the amount of main memory to use when sorting. */ - static void - specify_sort_size (int oi, char c, char const *s) -@@ -1596,7 +1745,7 @@ buffer_linelim (struct buffer const *buf - by KEY in LINE. */ - - static char * --begfield (struct line const *line, struct keyfield const *key) -+begfield_uni (const struct line *line, const struct keyfield *key) - { - char *ptr = line->text, *lim = ptr + line->length - 1; - size_t sword = key->sword; -@@ -1605,10 +1754,10 @@ begfield (struct line const *line, struc - /* The leading field separator itself is included in a field when -t - is absent. */ - -- if (tab != TAB_DEFAULT) -+ if (tab_length) - while (ptr < lim && sword--) - { -- while (ptr < lim && *ptr != tab) -+ while (ptr < lim && *ptr != tab[0]) - ++ptr; - if (ptr < lim) - ++ptr; -@@ -1634,11 +1783,70 @@ begfield (struct line const *line, struc - return ptr; - } - -+#if HAVE_MBRTOWC -+static char * -+begfield_mb (const struct line *line, const struct keyfield *key) -+{ -+ int i; -+ char *ptr = line->text, *lim = ptr + line->length - 1; -+ size_t sword = key->sword; -+ size_t schar = key->schar; -+ size_t mblength; -+ mbstate_t state; -+ -+ memset (&state, '\0', sizeof(mbstate_t)); -+ -+ if (tab_length) -+ while (ptr < lim && sword--) -+ { -+ while (ptr < lim && memcmp (ptr, tab, tab_length) != 0) -+ { -+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); -+ ptr += mblength; -+ } -+ if (ptr < lim) -+ { -+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); -+ ptr += mblength; -+ } -+ } -+ else -+ while (ptr < lim && sword--) -+ { -+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) -+ ptr += mblength; -+ if (ptr < lim) -+ { -+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); -+ ptr += mblength; -+ } -+ while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength)) -+ ptr += mblength; -+ } -+ -+ if (key->skipsblanks) -+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) -+ ptr += mblength; -+ -+ for (i = 0; i < schar; i++) -+ { -+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); -+ -+ if (ptr + mblength > lim) -+ break; -+ else -+ ptr += mblength; -+ } -+ -+ return ptr; -+} -+#endif -+ - /* Return the limit of (a pointer to the first character after) the field - in LINE specified by KEY. */ - - static char * --limfield (struct line const *line, struct keyfield const *key) -+limfield_uni (const struct line *line, const struct keyfield *key) - { - char *ptr = line->text, *lim = ptr + line->length - 1; - size_t eword = key->eword, echar = key->echar; -@@ -1653,10 +1861,10 @@ limfield (struct line const *line, struc - 'beginning' is the first character following the delimiting TAB. - Otherwise, leave PTR pointing at the first 'blank' character after - the preceding field. */ -- if (tab != TAB_DEFAULT) -+ if (tab_length) - while (ptr < lim && eword--) - { -- while (ptr < lim && *ptr != tab) -+ while (ptr < lim && *ptr != tab[0]) - ++ptr; - if (ptr < lim && (eword || echar)) - ++ptr; -@@ -1702,10 +1910,10 @@ limfield (struct line const *line, struc - */ - - /* Make LIM point to the end of (one byte past) the current field. */ -- if (tab != TAB_DEFAULT) -+ if (tab_length) - { - char *newlim; -- newlim = memchr (ptr, tab, lim - ptr); -+ newlim = memchr (ptr, tab[0], lim - ptr); - if (newlim) - lim = newlim; - } -@@ -1736,6 +1944,130 @@ limfield (struct line const *line, struc - return ptr; - } - -+#if HAVE_MBRTOWC -+static char * -+limfield_mb (const struct line *line, const struct keyfield *key) -+{ -+ char *ptr = line->text, *lim = ptr + line->length - 1; -+ size_t eword = key->eword, echar = key->echar; -+ int i; -+ size_t mblength; -+ mbstate_t state; -+ -+ if (echar == 0) -+ eword++; /* skip all of end field. */ -+ -+ memset (&state, '\0', sizeof(mbstate_t)); -+ -+ if (tab_length) -+ while (ptr < lim && eword--) -+ { -+ while (ptr < lim && memcmp (ptr, tab, tab_length) != 0) -+ { -+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); -+ ptr += mblength; -+ } -+ if (ptr < lim && (eword | echar)) -+ { -+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); -+ ptr += mblength; -+ } -+ } -+ else -+ while (ptr < lim && eword--) -+ { -+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) -+ ptr += mblength; -+ if (ptr < lim) -+ { -+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); -+ ptr += mblength; -+ } -+ while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength)) -+ ptr += mblength; -+ } -+ -+ -+# ifdef POSIX_UNSPECIFIED -+ /* Make LIM point to the end of (one byte past) the current field. */ -+ if (tab_length) -+ { -+ char *newlim, *p; -+ -+ newlim = NULL; -+ for (p = ptr; p < lim;) -+ { -+ if (memcmp (p, tab, tab_length) == 0) -+ { -+ newlim = p; -+ break; -+ } -+ -+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); -+ p += mblength; -+ } -+ } -+ else -+ { -+ char *newlim; -+ newlim = ptr; -+ -+ while (newlim < lim && ismbblank (newlim, lim - newlim, &mblength)) -+ newlim += mblength; -+ if (ptr < lim) -+ { -+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); -+ ptr += mblength; -+ } -+ while (newlim < lim && !ismbblank (newlim, lim - newlim, &mblength)) -+ newlim += mblength; -+ lim = newlim; -+ } -+# endif -+ -+ if (echar != 0) -+ { -+ /* If we're skipping leading blanks, don't start counting characters -+ * until after skipping past any leading blanks. */ -+ if (key->skipeblanks) -+ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) -+ ptr += mblength; -+ -+ memset (&state, '\0', sizeof(mbstate_t)); -+ -+ /* Advance PTR by ECHAR (if possible), but no further than LIM. */ -+ for (i = 0; i < echar; i++) -+ { -+ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); -+ -+ if (ptr + mblength > lim) -+ break; -+ else -+ ptr += mblength; -+ } -+ } -+ -+ return ptr; -+} -+#endif -+ -+static void -+skipblanks_uni (char **ptr, char *lim) -+{ -+ while (*ptr < lim && blanks[to_uchar (**ptr)]) -+ ++(*ptr); -+} -+ -+#if HAVE_MBRTOWC -+static void -+skipblanks_mb (char **ptr, char *lim) -+{ -+ size_t mblength; -+ while (*ptr < lim && ismbblank (*ptr, lim - *ptr, &mblength)) -+ (*ptr) += mblength; -+} -+#endif -+ - /* Fill BUF reading from FP, moving buf->left bytes from the end - of buf->buf to the beginning first. If EOF is reached and the - file wasn't terminated by a newline, supply one. Set up BUF's line -@@ -1822,8 +2154,22 @@ fillbuf (struct buffer *buf, FILE *fp, c - else - { - if (key->skipsblanks) -- while (blanks[to_uchar (*line_start)]) -- line_start++; -+ { -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ size_t mblength; -+ while (line_start < line->keylim && -+ ismbblank (line_start, -+ line->keylim - line_start, -+ &mblength)) -+ line_start += mblength; -+ } -+ else -+#endif -+ while (blanks[to_uchar (*line_start)]) -+ line_start++; -+ } - line->keybeg = line_start; - } - } -@@ -1944,7 +2290,7 @@ human_numcompare (char const *a, char co - hideously fast. */ - - static int --numcompare (char const *a, char const *b) -+numcompare_uni (const char *a, const char *b) - { - while (blanks[to_uchar (*a)]) - a++; -@@ -1954,6 +2300,25 @@ numcompare (char const *a, char const *b - return strnumcmp (a, b, decimal_point, thousands_sep); - } - -+#if HAVE_MBRTOWC -+static int -+numcompare_mb (const char *a, const char *b) -+{ -+ size_t mblength, len; -+ len = strlen (a); /* okay for UTF-8 */ -+ while (*a && ismbblank (a, len > MB_CUR_MAX ? MB_CUR_MAX : len, &mblength)) -+ { -+ a += mblength; -+ len -= mblength; -+ } -+ len = strlen (b); /* okay for UTF-8 */ -+ while (*b && ismbblank (b, len > MB_CUR_MAX ? MB_CUR_MAX : len, &mblength)) -+ b += mblength; -+ -+ return strnumcmp (a, b, decimal_point, thousands_sep); -+} -+#endif /* HAV_EMBRTOWC */ -+ - /* Work around a problem whereby the long double value returned by glibc's - strtold ("NaN", ...) contains uninitialized bits: clear all bytes of - A and B before calling strtold. FIXME: remove this function once -@@ -2004,7 +2369,7 @@ general_numcompare (char const *sa, char - Return 0 if the name in S is not recognized. */ - - static int --getmonth (char const *month, char **ea) -+getmonth_uni (char const *month, size_t len, char **ea) - { - size_t lo = 0; - size_t hi = MONTHS_PER_YEAR; -@@ -2280,15 +2645,14 @@ debug_key (struct line const *line, stru - char saved = *lim; - *lim = '\0'; - -- while (blanks[to_uchar (*beg)]) -- beg++; -+ skipblanks (&beg, lim); - - char *tighter_lim = beg; - - if (lim < beg) - tighter_lim = lim; - else if (key->month) -- getmonth (beg, &tighter_lim); -+ getmonth (beg, lim-beg, &tighter_lim); - else if (key->general_numeric) - ignore_value (strtold (beg, &tighter_lim)); - else if (key->numeric || key->human_numeric) -@@ -2432,7 +2796,7 @@ key_warnings (struct keyfield const *gke - bool maybe_space_aligned = !hard_LC_COLLATE && default_key_compare (key) - && !(key->schar || key->echar); - bool line_offset = key->eword == 0 && key->echar != 0; /* -k1.x,1.y */ -- if (!gkey_only && tab == TAB_DEFAULT && !line_offset -+ if (!gkey_only && !tab_length && !line_offset - && ((!key->skipsblanks && !(implicit_skip || maybe_space_aligned)) - || (!key->skipsblanks && key->schar) - || (!key->skipeblanks && key->echar))) -@@ -2490,11 +2854,87 @@ key_warnings (struct keyfield const *gke - error (0, 0, _("option '-r' only applies to last-resort comparison")); - } - -+#if HAVE_MBRTOWC -+static int -+getmonth_mb (const char *s, size_t len, char **ea) -+{ -+ char *month; -+ register size_t i; -+ register int lo = 0, hi = MONTHS_PER_YEAR, result; -+ char *tmp; -+ size_t wclength, mblength; -+ const char *pp; -+ const wchar_t *wpp; -+ wchar_t *month_wcs; -+ mbstate_t state; -+ -+ while (len > 0 && ismbblank (s, len, &mblength)) -+ { -+ s += mblength; -+ len -= mblength; -+ } -+ -+ if (len == 0) -+ return 0; -+ -+ if (SIZE_MAX - len < 1) -+ xalloc_die (); -+ -+ month = (char *) xnmalloc (len + 1, MB_CUR_MAX); -+ -+ pp = tmp = (char *) xnmalloc (len + 1, MB_CUR_MAX); -+ memcpy (tmp, s, len); -+ tmp[len] = '\0'; -+ wpp = month_wcs = (wchar_t *) xnmalloc (len + 1, sizeof (wchar_t)); -+ memset (&state, '\0', sizeof (mbstate_t)); -+ -+ wclength = mbsrtowcs (month_wcs, &pp, len + 1, &state); -+ if (wclength == (size_t)-1 || pp != NULL) -+ error (SORT_FAILURE, 0, _("Invalid multibyte input %s."), quote(s)); -+ -+ for (i = 0; i < wclength; i++) -+ { -+ month_wcs[i] = towupper(month_wcs[i]); -+ if (iswblank (month_wcs[i])) -+ { -+ month_wcs[i] = L'\0'; -+ break; -+ } -+ } -+ -+ mblength = wcsrtombs (month, &wpp, (len + 1) * MB_CUR_MAX, &state); -+ assert (mblength != (-1) && wpp == NULL); -+ -+ do -+ { -+ int ix = (lo + hi) / 2; -+ -+ if (strncmp (month, monthtab[ix].name, strlen (monthtab[ix].name)) < 0) -+ hi = ix; -+ else -+ lo = ix; -+ } -+ while (hi - lo > 1); -+ -+ result = (!strncmp (month, monthtab[lo].name, strlen (monthtab[lo].name)) -+ ? monthtab[lo].val : 0); -+ -+ if (ea && result) -+ *ea = (char*) s + strlen (monthtab[lo].name); -+ -+ free (month); -+ free (tmp); -+ free (month_wcs); -+ -+ return result; -+} -+#endif -+ - /* Compare two lines A and B trying every key in sequence until there - are no more keys or a difference is found. */ - - static int --keycompare (struct line const *a, struct line const *b) -+keycompare_uni (const struct line *a, const struct line *b) - { - struct keyfield *key = keylist; - -@@ -2579,7 +3019,7 @@ keycompare (struct line const *a, struct - else if (key->human_numeric) - diff = human_numcompare (ta, tb); - else if (key->month) -- diff = getmonth (ta, NULL) - getmonth (tb, NULL); -+ diff = getmonth (ta, tlena, NULL) - getmonth (tb, tlenb, NULL); - else if (key->random) - diff = compare_random (ta, tlena, tb, tlenb); - else if (key->version) -@@ -2695,6 +3135,211 @@ keycompare (struct line const *a, struct - return key->reverse ? -diff : diff; - } - -+#if HAVE_MBRTOWC -+static int -+keycompare_mb (const struct line *a, const struct line *b) -+{ -+ struct keyfield *key = keylist; -+ -+ /* For the first iteration only, the key positions have been -+ precomputed for us. */ -+ char *texta = a->keybeg; -+ char *textb = b->keybeg; -+ char *lima = a->keylim; -+ char *limb = b->keylim; -+ -+ size_t mblength_a, mblength_b; -+ wchar_t wc_a, wc_b; -+ mbstate_t state_a, state_b; -+ -+ int diff = 0; -+ -+ memset (&state_a, '\0', sizeof(mbstate_t)); -+ memset (&state_b, '\0', sizeof(mbstate_t)); -+ /* Ignore keys with start after end. */ -+ if (a->keybeg - a->keylim > 0) -+ return 0; -+ -+ -+ /* Ignore and/or translate chars before comparing. */ -+# define IGNORE_CHARS(NEW_LEN, LEN, TEXT, COPY, WC, MBLENGTH, STATE) \ -+ do \ -+ { \ -+ wchar_t uwc; \ -+ char mbc[MB_LEN_MAX]; \ -+ mbstate_t state_wc; \ -+ \ -+ for (NEW_LEN = i = 0; i < LEN;) \ -+ { \ -+ mbstate_t state_bak; \ -+ \ -+ state_bak = STATE; \ -+ MBLENGTH = mbrtowc (&WC, TEXT + i, LEN - i, &STATE); \ -+ \ -+ if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1 \ -+ || MBLENGTH == 0) \ -+ { \ -+ if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1) \ -+ STATE = state_bak; \ -+ if (!ignore) \ -+ COPY[NEW_LEN++] = TEXT[i]; \ -+ i++; \ -+ continue; \ -+ } \ -+ \ -+ if (ignore) \ -+ { \ -+ if ((ignore == nonprinting && !iswprint (WC)) \ -+ || (ignore == nondictionary \ -+ && !iswalnum (WC) && !iswblank (WC))) \ -+ { \ -+ i += MBLENGTH; \ -+ continue; \ -+ } \ -+ } \ -+ \ -+ if (translate) \ -+ { \ -+ \ -+ uwc = towupper(WC); \ -+ if (WC == uwc) \ -+ { \ -+ memcpy (mbc, TEXT + i, MBLENGTH); \ -+ i += MBLENGTH; \ -+ } \ -+ else \ -+ { \ -+ i += MBLENGTH; \ -+ WC = uwc; \ -+ memset (&state_wc, '\0', sizeof (mbstate_t)); \ -+ \ -+ MBLENGTH = wcrtomb (mbc, WC, &state_wc); \ -+ assert (MBLENGTH != (size_t)-1 && MBLENGTH != 0); \ -+ } \ -+ \ -+ for (j = 0; j < MBLENGTH; j++) \ -+ COPY[NEW_LEN++] = mbc[j]; \ -+ } \ -+ else \ -+ for (j = 0; j < MBLENGTH; j++) \ -+ COPY[NEW_LEN++] = TEXT[i++]; \ -+ } \ -+ COPY[NEW_LEN] = '\0'; \ -+ } \ -+ while (0) -+ -+ /* Actually compare the fields. */ -+ -+ for (;;) -+ { -+ /* Find the lengths. */ -+ size_t lena = lima <= texta ? 0 : lima - texta; -+ size_t lenb = limb <= textb ? 0 : limb - textb; -+ -+ char enda IF_LINT (= 0); -+ char endb IF_LINT (= 0); -+ -+ char const *translate = key->translate; -+ bool const *ignore = key->ignore; -+ -+ if (ignore || translate) -+ { -+ if (SIZE_MAX - lenb - 2 < lena) -+ xalloc_die (); -+ char *copy_a = (char *) xnmalloc (lena + lenb + 2, MB_CUR_MAX); -+ char *copy_b = copy_a + lena * MB_CUR_MAX + 1; -+ size_t new_len_a, new_len_b; -+ size_t i, j; -+ -+ IGNORE_CHARS (new_len_a, lena, texta, copy_a, -+ wc_a, mblength_a, state_a); -+ IGNORE_CHARS (new_len_b, lenb, textb, copy_b, -+ wc_b, mblength_b, state_b); -+ texta = copy_a; textb = copy_b; -+ lena = new_len_a; lenb = new_len_b; -+ } -+ else -+ { -+ /* Use the keys in-place, temporarily null-terminated. */ -+ enda = texta[lena]; texta[lena] = '\0'; -+ endb = textb[lenb]; textb[lenb] = '\0'; -+ } -+ -+ if (key->random) -+ diff = compare_random (texta, lena, textb, lenb); -+ else if (key->numeric | key->general_numeric | key->human_numeric) -+ { -+ char savea = *lima, saveb = *limb; -+ -+ *lima = *limb = '\0'; -+ diff = (key->numeric ? numcompare (texta, textb) -+ : key->general_numeric ? general_numcompare (texta, textb) -+ : human_numcompare (texta, textb)); -+ *lima = savea, *limb = saveb; -+ } -+ else if (key->version) -+ diff = filevercmp (texta, textb); -+ else if (key->month) -+ diff = getmonth (texta, lena, NULL) - getmonth (textb, lenb, NULL); -+ else if (lena == 0) -+ diff = - NONZERO (lenb); -+ else if (lenb == 0) -+ diff = 1; -+ else if (hard_LC_COLLATE && !folding) -+ { -+ diff = xmemcoll0 (texta, lena + 1, textb, lenb + 1); -+ } -+ else -+ { -+ diff = memcmp (texta, textb, MIN (lena, lenb)); -+ if (diff == 0) -+ diff = lena < lenb ? -1 : lena != lenb; -+ } -+ -+ if (ignore || translate) -+ free (texta); -+ else -+ { -+ texta[lena] = enda; -+ textb[lenb] = endb; -+ } -+ -+ if (diff) -+ goto not_equal; -+ -+ key = key->next; -+ if (! key) -+ break; -+ -+ /* Find the beginning and limit of the next field. */ -+ if (key->eword != -1) -+ lima = limfield (a, key), limb = limfield (b, key); -+ else -+ lima = a->text + a->length - 1, limb = b->text + b->length - 1; -+ -+ if (key->sword != -1) -+ texta = begfield (a, key), textb = begfield (b, key); -+ else -+ { -+ texta = a->text, textb = b->text; -+ if (key->skipsblanks) -+ { -+ while (texta < lima && ismbblank (texta, lima - texta, &mblength_a)) -+ texta += mblength_a; -+ while (textb < limb && ismbblank (textb, limb - textb, &mblength_b)) -+ textb += mblength_b; -+ } -+ } -+ } -+ -+not_equal: -+ if (key && key->reverse) -+ return -diff; -+ else -+ return diff; -+} -+#endif -+ - /* Compare two lines A and B, returning negative, zero, or positive - depending on whether A compares less than, equal to, or greater than B. */ - -@@ -2722,7 +3367,7 @@ compare (struct line const *a, struct li - diff = - NONZERO (blen); - else if (blen == 0) - diff = 1; -- else if (hard_LC_COLLATE) -+ else if (hard_LC_COLLATE && !folding) - { - /* Note xmemcoll0 is a performance enhancement as - it will not unconditionally write '\0' after the -@@ -4121,6 +4766,7 @@ set_ordering (char const *s, struct keyf - break; - case 'f': - key->translate = fold_toupper; -+ folding = true; - break; - case 'g': - key->general_numeric = true; -@@ -4199,7 +4845,7 @@ main (int argc, char **argv) - initialize_exit_failure (SORT_FAILURE); - - hard_LC_COLLATE = hard_locale (LC_COLLATE); --#if HAVE_NL_LANGINFO -+#if HAVE_LANGINFO_CODESET - hard_LC_TIME = hard_locale (LC_TIME); - #endif - -@@ -4220,6 +4866,29 @@ main (int argc, char **argv) - thousands_sep = -1; - } - -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ inittables = inittables_mb; -+ begfield = begfield_mb; -+ limfield = limfield_mb; -+ skipblanks = skipblanks_mb; -+ getmonth = getmonth_mb; -+ keycompare = keycompare_mb; -+ numcompare = numcompare_mb; -+ } -+ else -+#endif -+ { -+ inittables = inittables_uni; -+ begfield = begfield_uni; -+ limfield = limfield_uni; -+ skipblanks = skipblanks_uni; -+ getmonth = getmonth_uni; -+ keycompare = keycompare_uni; -+ numcompare = numcompare_uni; -+ } -+ - have_read_stdin = false; - inittables (); - -@@ -4494,13 +5163,34 @@ main (int argc, char **argv) - - case 't': - { -- char newtab = optarg[0]; -- if (! newtab) -+ char newtab[MB_LEN_MAX + 1]; -+ size_t newtab_length = 1; -+ strncpy (newtab, optarg, MB_LEN_MAX); -+ if (! newtab[0]) - error (SORT_FAILURE, 0, _("empty tab")); -- if (optarg[1]) -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ wchar_t wc; -+ mbstate_t state; -+ -+ memset (&state, '\0', sizeof (mbstate_t)); -+ newtab_length = mbrtowc (&wc, newtab, strnlen (newtab, -+ MB_LEN_MAX), -+ &state); -+ switch (newtab_length) -+ { -+ case (size_t) -1: -+ case (size_t) -2: -+ case 0: -+ newtab_length = 1; -+ } -+ } -+#endif -+ if (newtab_length == 1 && optarg[1]) - { - if (STREQ (optarg, "\\0")) -- newtab = '\0'; -+ newtab[0] = '\0'; - else - { - /* Provoke with 'sort -txx'. Complain about -@@ -4511,9 +5201,12 @@ main (int argc, char **argv) - quote (optarg)); - } - } -- if (tab != TAB_DEFAULT && tab != newtab) -+ if (tab_length -+ && (tab_length != newtab_length -+ || memcmp (tab, newtab, tab_length) != 0)) - error (SORT_FAILURE, 0, _("incompatible tabs")); -- tab = newtab; -+ memcpy (tab, newtab, newtab_length); -+ tab_length = newtab_length; - } - break; - -@@ -4751,12 +5444,10 @@ main (int argc, char **argv) - sort (files, nfiles, outfile, nthreads); - } - --#ifdef lint - if (files_from) - readtokens0_free (&tok); - else - free (files); --#endif - - if (have_read_stdin && fclose (stdin) == EOF) - die (_("close failed"), "-"); -diff -Naurp coreutils-8.25-orig/src/unexpand.c coreutils-8.25/src/unexpand.c ---- coreutils-8.25-orig/src/unexpand.c 2016-01-01 07:48:50.000000000 -0600 -+++ coreutils-8.25/src/unexpand.c 2016-02-08 19:07:10.311944651 -0600 -@@ -38,12 +38,29 @@ - #include - #include - #include -+ -+/* Get mbstate_t, mbrtowc(), wcwidth(). */ -+#if HAVE_WCHAR_H -+# include -+#endif -+ - #include "system.h" - #include "error.h" - #include "fadvise.h" - #include "quote.h" - #include "xstrndup.h" - -+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ -+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 -+# define MB_LEN_MAX 16 -+#endif -+ -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif -+ - /* The official name of this program (e.g., no 'g' prefix). */ - #define PROGRAM_NAME "unexpand" - -@@ -103,6 +120,210 @@ static struct option const longopts[] = - {NULL, 0, NULL, 0} - }; - -+static FILE *next_file (FILE *fp); -+ -+#if HAVE_MBRTOWC -+static void -+unexpand_multibyte (void) -+{ -+ FILE *fp; /* Input stream. */ -+ mbstate_t i_state; /* Current shift state of the input stream. */ -+ mbstate_t i_state_bak; /* Back up the I_STATE. */ -+ mbstate_t o_state; /* Current shift state of the output stream. */ -+ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ -+ char *bufpos = buf; /* Next read position of BUF. */ -+ size_t buflen = 0; /* The length of the byte sequence in buf. */ -+ wint_t wc; /* A gotten wide character. */ -+ size_t mblength; /* The byte size of a multibyte character -+ which shows as same character as WC. */ -+ bool prev_tab = false; -+ -+ /* Index in `tab_list' of next tabstop: */ -+ int tab_index = 0; /* For calculating width of pending tabs. */ -+ int print_tab_index = 0; /* For printing as many tabs as possible. */ -+ unsigned int column = 0; /* Column on screen of next char. */ -+ int next_tab_column; /* Column the next tab stop is on. */ -+ int convert = 1; /* If nonzero, perform translations. */ -+ unsigned int pending = 0; /* Pending columns of blanks. */ -+ -+ fp = next_file ((FILE *) NULL); -+ if (fp == NULL) -+ return; -+ -+ memset (&o_state, '\0', sizeof(mbstate_t)); -+ memset (&i_state, '\0', sizeof(mbstate_t)); -+ -+ for (;;) -+ { -+ if (buflen < MB_LEN_MAX && !feof(fp) && !ferror(fp)) -+ { -+ memmove (buf, bufpos, buflen); -+ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, fp); -+ bufpos = buf; -+ } -+ -+ /* Get a wide character. */ -+ if (buflen < 1) -+ { -+ mblength = 1; -+ wc = WEOF; -+ } -+ else -+ { -+ i_state_bak = i_state; -+ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &i_state); -+ } -+ -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ i_state = i_state_bak; -+ wc = L'\0'; -+ } -+ -+ if (wc == L' ' && convert && column < INT_MAX) -+ { -+ ++pending; -+ ++column; -+ } -+ else if (wc == L'\t' && convert) -+ { -+ if (tab_size == 0) -+ { -+ /* Do not let tab_index == first_free_tab; -+ stop when it is 1 less. */ -+ while (tab_index < first_free_tab - 1 -+ && column >= tab_list[tab_index]) -+ tab_index++; -+ next_tab_column = tab_list[tab_index]; -+ if (tab_index < first_free_tab - 1) -+ tab_index++; -+ if (column >= next_tab_column) -+ { -+ convert = 0; /* Ran out of tab stops. */ -+ goto flush_pend_mb; -+ } -+ } -+ else -+ { -+ next_tab_column = column + tab_size - column % tab_size; -+ } -+ pending += next_tab_column - column; -+ column = next_tab_column; -+ } -+ else -+ { -+flush_pend_mb: -+ /* Flush pending spaces. Print as many tabs as possible, -+ then print the rest as spaces. */ -+ if (pending == 1 && column != 1 && !prev_tab) -+ { -+ putchar (' '); -+ pending = 0; -+ } -+ column -= pending; -+ while (pending > 0) -+ { -+ if (tab_size == 0) -+ { -+ /* Do not let print_tab_index == first_free_tab; -+ stop when it is 1 less. */ -+ while (print_tab_index < first_free_tab - 1 -+ && column >= tab_list[print_tab_index]) -+ print_tab_index++; -+ next_tab_column = tab_list[print_tab_index]; -+ if (print_tab_index < first_free_tab - 1) -+ print_tab_index++; -+ } -+ else -+ { -+ next_tab_column = -+ column + tab_size - column % tab_size; -+ } -+ if (next_tab_column - column <= pending) -+ { -+ putchar ('\t'); -+ pending -= next_tab_column - column; -+ column = next_tab_column; -+ } -+ else -+ { -+ --print_tab_index; -+ column += pending; -+ while (pending != 0) -+ { -+ putchar (' '); -+ pending--; -+ } -+ } -+ } -+ -+ if (wc == WEOF) -+ { -+ fp = next_file (fp); -+ if (fp == NULL) -+ break; /* No more files. */ -+ else -+ { -+ memset (&i_state, '\0', sizeof(mbstate_t)); -+ continue; -+ } -+ } -+ -+ if (mblength == (size_t)-1 || mblength == (size_t)-2) -+ { -+ if (convert) -+ { -+ ++column; -+ if (convert_entire_line == 0) -+ convert = 0; -+ } -+ mblength = 1; -+ putchar (buf[0]); -+ } -+ else if (mblength == 0) -+ { -+ if (convert && convert_entire_line == 0) -+ convert = 0; -+ mblength = 1; -+ putchar ('\0'); -+ } -+ else -+ { -+ if (convert) -+ { -+ if (wc == L'\b') -+ { -+ if (column > 0) -+ --column; -+ } -+ else -+ { -+ int width; /* The width of WC. */ -+ -+ width = wcwidth (wc); -+ column += (width > 0) ? width : 0; -+ if (convert_entire_line == 0) -+ convert = 0; -+ } -+ } -+ -+ if (wc == L'\n') -+ { -+ tab_index = print_tab_index = 0; -+ column = pending = 0; -+ convert = 1; -+ } -+ fwrite (bufpos, sizeof(char), mblength, stdout); -+ } -+ } -+ prev_tab = wc == L'\t'; -+ buflen -= mblength; -+ bufpos += mblength; -+ } -+} -+#endif -+ -+ - void - usage (int status) - { -@@ -523,7 +744,12 @@ main (int argc, char **argv) - - file_list = (optind < argc ? &argv[optind] : stdin_argv); - -- unexpand (); -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ unexpand_multibyte (); -+ else -+#endif -+ unexpand (); - - if (have_read_stdin && fclose (stdin) != 0) - error (EXIT_FAILURE, errno, "-"); -diff -Naurp coreutils-8.25-orig/src/uniq.c coreutils-8.25/src/uniq.c ---- coreutils-8.25-orig/src/uniq.c 2016-01-13 05:08:59.000000000 -0600 -+++ coreutils-8.25/src/uniq.c 2016-02-08 19:07:10.312944654 -0600 -@@ -21,6 +21,17 @@ - #include - #include - -+/* Get mbstate_t, mbrtowc(). */ -+#if HAVE_WCHAR_H -+# include -+#endif -+ -+/* Get isw* functions. */ -+#if HAVE_WCTYPE_H -+# include -+#endif -+#include -+ - #include "system.h" - #include "argmatch.h" - #include "linebuffer.h" -@@ -33,6 +44,18 @@ - #include "xstrtol.h" - #include "memcasecmp.h" - #include "quote.h" -+#include "xmemcoll.h" -+ -+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC -+ installation; work around this configuration error. */ -+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 -+# define MB_LEN_MAX 16 -+#endif -+ -+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ -+#if HAVE_MBRTOWC && defined mbstate_t -+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) -+#endif - - /* The official name of this program (e.g., no 'g' prefix). */ - #define PROGRAM_NAME "uniq" -@@ -143,6 +166,10 @@ enum - GROUP_OPTION = CHAR_MAX + 1 - }; - -+/* Function pointers. */ -+static char * -+(*find_field) (struct linebuffer *line); -+ - static struct option const longopts[] = - { - {"count", no_argument, NULL, 'c'}, -@@ -252,7 +279,7 @@ size_opt (char const *opt, char const *m - return a pointer to the beginning of the line's field to be compared. */ - - static char * _GL_ATTRIBUTE_PURE --find_field (struct linebuffer const *line) -+find_field_uni (struct linebuffer *line) - { - size_t count; - char const *lp = line->buffer; -@@ -272,6 +299,83 @@ find_field (struct linebuffer const *lin - return line->buffer + i; - } - -+#if HAVE_MBRTOWC -+ -+# define MBCHAR_TO_WCHAR(WC, MBLENGTH, LP, POS, SIZE, STATEP, CONVFAIL) \ -+ do \ -+ { \ -+ mbstate_t state_bak; \ -+ \ -+ CONVFAIL = 0; \ -+ state_bak = *STATEP; \ -+ \ -+ MBLENGTH = mbrtowc (&WC, LP + POS, SIZE - POS, STATEP); \ -+ \ -+ switch (MBLENGTH) \ -+ { \ -+ case (size_t)-2: \ -+ case (size_t)-1: \ -+ *STATEP = state_bak; \ -+ CONVFAIL++; \ -+ /* Fall through */ \ -+ case 0: \ -+ MBLENGTH = 1; \ -+ } \ -+ } \ -+ while (0) -+ -+static char * -+find_field_multi (struct linebuffer *line) -+{ -+ size_t count; -+ char *lp = line->buffer; -+ size_t size = line->length - 1; -+ size_t pos; -+ size_t mblength; -+ wchar_t wc; -+ mbstate_t *statep; -+ int convfail = 0; -+ -+ pos = 0; -+ statep = &(line->state); -+ -+ /* skip fields. */ -+ for (count = 0; count < skip_fields && pos < size; count++) -+ { -+ while (pos < size) -+ { -+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); -+ -+ if (convfail || !(iswblank (wc) || wc == '\n')) -+ { -+ pos += mblength; -+ break; -+ } -+ pos += mblength; -+ } -+ -+ while (pos < size) -+ { -+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); -+ -+ if (!convfail && (iswblank (wc) || wc == '\n')) -+ break; -+ -+ pos += mblength; -+ } -+ } -+ -+ /* skip fields. */ -+ for (count = 0; count < skip_chars && pos < size; count++) -+ { -+ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); -+ pos += mblength; -+ } -+ -+ return lp + pos; -+} -+#endif -+ - /* Return false if two strings OLD and NEW match, true if not. - OLD and NEW point not to the beginnings of the lines - but rather to the beginnings of the fields to compare. -@@ -280,6 +384,8 @@ find_field (struct linebuffer const *lin - static bool - different (char *old, char *new, size_t oldlen, size_t newlen) - { -+ char *copy_old, *copy_new; -+ - if (check_chars < oldlen) - oldlen = check_chars; - if (check_chars < newlen) -@@ -287,15 +393,104 @@ different (char *old, char *new, size_t - - if (ignore_case) - { -- /* FIXME: This should invoke strcoll somehow. */ -- return oldlen != newlen || memcasecmp (old, new, oldlen); -+ size_t i; -+ -+ copy_old = xmalloc (oldlen + 1); -+ copy_new = xmalloc (oldlen + 1); -+ -+ for (i = 0; i < oldlen; i++) -+ { -+ copy_old[i] = toupper (old[i]); -+ copy_new[i] = toupper (new[i]); -+ } -+ bool rc = xmemcoll (copy_old, oldlen, copy_new, newlen); -+ free (copy_old); -+ free (copy_new); -+ return rc; - } -- else if (hard_LC_COLLATE) -- return xmemcoll (old, oldlen, new, newlen) != 0; - else -- return oldlen != newlen || memcmp (old, new, oldlen); -+ { -+ copy_old = (char *)old; -+ copy_new = (char *)new; -+ } -+ -+ return xmemcoll (copy_old, oldlen, copy_new, newlen); -+ - } - -+#if HAVE_MBRTOWC -+static int -+different_multi (const char *old, const char *new, size_t oldlen, size_t newlen, mbstate_t oldstate, mbstate_t newstate) -+{ -+ size_t i, j, chars; -+ const char *str[2]; -+ char *copy[2]; -+ size_t len[2]; -+ mbstate_t state[2]; -+ size_t mblength; -+ wchar_t wc, uwc; -+ mbstate_t state_bak; -+ -+ str[0] = old; -+ str[1] = new; -+ len[0] = oldlen; -+ len[1] = newlen; -+ state[0] = oldstate; -+ state[1] = newstate; -+ -+ for (i = 0; i < 2; i++) -+ { -+ copy[i] = xmalloc (len[i] + 1); -+ memset (copy[i], '\0', len[i] + 1); -+ -+ for (j = 0, chars = 0; j < len[i] && chars < check_chars; chars++) -+ { -+ state_bak = state[i]; -+ mblength = mbrtowc (&wc, str[i] + j, len[i] - j, &(state[i])); -+ -+ switch (mblength) -+ { -+ case (size_t)-1: -+ case (size_t)-2: -+ state[i] = state_bak; -+ /* Fall through */ -+ case 0: -+ mblength = 1; -+ break; -+ -+ default: -+ if (ignore_case) -+ { -+ uwc = towupper (wc); -+ -+ if (uwc != wc) -+ { -+ mbstate_t state_wc; -+ size_t mblen; -+ -+ memset (&state_wc, '\0', sizeof(mbstate_t)); -+ mblen = wcrtomb (copy[i] + j, uwc, &state_wc); -+ assert (mblen != (size_t)-1); -+ } -+ else -+ memcpy (copy[i] + j, str[i] + j, mblength); -+ } -+ else -+ memcpy (copy[i] + j, str[i] + j, mblength); -+ } -+ j += mblength; -+ } -+ copy[i][j] = '\0'; -+ len[i] = j; -+ } -+ int rc = xmemcoll (copy[0], len[0], copy[1], len[1]); -+ free (copy[0]); -+ free (copy[1]); -+ return rc; -+ -+} -+#endif -+ - /* Output the line in linebuffer LINE to standard output - provided that the switches say it should be output. - MATCH is true if the line matches the previous line. -@@ -359,19 +554,38 @@ check_file (const char *infile, const ch - char *prevfield IF_LINT ( = NULL); - size_t prevlen IF_LINT ( = 0); - bool first_group_printed = false; -+#if HAVE_MBRTOWC -+ mbstate_t prevstate; -+ -+ memset (&prevstate, '\0', sizeof (mbstate_t)); -+#endif - - while (!feof (stdin)) - { - char *thisfield; - size_t thislen; - bool new_group; -+#if HAVE_MBRTOWC -+ mbstate_t thisstate; -+#endif - - if (readlinebuffer_delim (thisline, stdin, delimiter) == 0) - break; - - thisfield = find_field (thisline); - thislen = thisline->length - 1 - (thisfield - thisline->buffer); -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ thisstate = thisline->state; - -+ new_group = (prevline->length == 0 -+ || different_multi (thisfield, prevfield, -+ thislen, prevlen, -+ thisstate, prevstate)); -+ } -+ else -+#endif - new_group = (prevline->length == 0 - || different (thisfield, prevfield, thislen, prevlen)); - -@@ -389,6 +603,10 @@ check_file (const char *infile, const ch - SWAP_LINES (prevline, thisline); - prevfield = thisfield; - prevlen = thislen; -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ prevstate = thisstate; -+#endif - first_group_printed = true; - } - } -@@ -401,17 +619,26 @@ check_file (const char *infile, const ch - size_t prevlen; - uintmax_t match_count = 0; - bool first_delimiter = true; -+#if HAVE_MBRTOWC -+ mbstate_t prevstate; -+#endif - - if (readlinebuffer_delim (prevline, stdin, delimiter) == 0) - goto closefiles; - prevfield = find_field (prevline); - prevlen = prevline->length - 1 - (prevfield - prevline->buffer); -+#if HAVE_MBRTOWC -+ prevstate = prevline->state; -+#endif - - while (!feof (stdin)) - { - bool match; - char *thisfield; - size_t thislen; -+#if HAVE_MBRTOWC -+ mbstate_t thisstate = thisline->state; -+#endif - if (readlinebuffer_delim (thisline, stdin, delimiter) == 0) - { - if (ferror (stdin)) -@@ -420,6 +647,14 @@ check_file (const char *infile, const ch - } - thisfield = find_field (thisline); - thislen = thisline->length - 1 - (thisfield - thisline->buffer); -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ match = !different_multi (thisfield, prevfield, -+ thislen, prevlen, thisstate, prevstate); -+ } -+ else -+#endif - match = !different (thisfield, prevfield, thislen, prevlen); - match_count += match; - -@@ -452,6 +687,9 @@ check_file (const char *infile, const ch - SWAP_LINES (prevline, thisline); - prevfield = thisfield; - prevlen = thislen; -+#if HAVE_MBRTOWC -+ prevstate = thisstate; -+#endif - if (!match) - match_count = 0; - } -@@ -498,6 +736,19 @@ main (int argc, char **argv) - - atexit (close_stdout); - -+#if HAVE_MBRTOWC -+ if (MB_CUR_MAX > 1) -+ { -+ find_field = find_field_multi; -+ } -+ else -+#endif -+ { -+ find_field = find_field_uni; -+ } -+ -+ -+ - skip_chars = 0; - skip_fields = 0; - check_chars = SIZE_MAX; -diff -Naurp coreutils-8.25-orig/tests/i18n/sort-month.sh coreutils-8.25/tests/i18n/sort-month.sh ---- coreutils-8.25-orig/tests/i18n/sort-month.sh 1969-12-31 18:00:00.000000000 -0600 -+++ coreutils-8.25/tests/i18n/sort-month.sh 2016-02-08 19:07:10.312944654 -0600 -@@ -0,0 +1,34 @@ -+#!/bin/sh -+# Verify sort -M multi-byte support. -+ -+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src -+print_ver_ sort -+require_valgrind_ -+ -+# Skip this test if some deallocations are -+# avoided at process end. -+grep '^#define lint 1' $CONFIG_HEADER > /dev/null || -+ skip_ 'Allocation checks only work reliably in "lint" mode' -+ -+export LC_ALL=en_US.UTF-8 -+locale -k LC_CTYPE | grep -q "charmap.*UTF-8" \ -+ || skip_ "No UTF-8 locale available" -+ -+# Note the use of ɑ here which expands to -+# a wider representation upon case conversion -+# which triggered an assertion in sort -M -+cat < exp -+. -+ɑ -+EOF -+ -+ -+# check large mem leak with --month-sort -+# https://bugzilla.redhat.com/show_bug.cgi?id=1259942 -+valgrind --leak-check=full \ -+ --error-exitcode=1 --errors-for-leak-kinds=definite \ -+ sort -M < exp > out || fail=1 -+compare exp out || { fail=1; cat out; } -+ -+ -+Exit $fail -diff -Naurp coreutils-8.25-orig/tests/i18n/sort.sh coreutils-8.25/tests/i18n/sort.sh ---- coreutils-8.25-orig/tests/i18n/sort.sh 1969-12-31 18:00:00.000000000 -0600 -+++ coreutils-8.25/tests/i18n/sort.sh 2016-02-08 19:07:10.312944654 -0600 -@@ -0,0 +1,29 @@ -+#!/bin/sh -+# Verify sort's multi-byte support. -+ -+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src -+print_ver_ sort -+ -+export LC_ALL=en_US.UTF-8 -+locale -k LC_CTYPE | grep -q "charmap.*UTF-8" \ -+ || skip_ "No UTF-8 locale available" -+ -+# Enable heap consistency checkng on older systems -+export MALLOC_CHECK_=2 -+ -+ -+# check buffer overflow issue due to -+# expanding multi-byte representation due to case conversion -+# https://bugzilla.suse.com/show_bug.cgi?id=928749 -+cat < exp -+. -+ɑ -+EOF -+cat < out || fail=1 -+. -+ɑ -+EOF -+compare exp out || { fail=1; cat out; } -+ -+ -+Exit $fail -diff -Naurp coreutils-8.25-orig/tests/local.mk coreutils-8.25/tests/local.mk ---- coreutils-8.25-orig/tests/local.mk 2016-01-16 12:18:13.000000000 -0600 -+++ coreutils-8.25/tests/local.mk 2016-02-08 19:07:10.313944658 -0600 -@@ -344,6 +344,9 @@ all_tests = \ - tests/misc/sort-discrim.sh \ - tests/misc/sort-files0-from.pl \ - tests/misc/sort-float.sh \ -+ tests/misc/sort-mb-tests.sh \ -+ tests/i18n/sort.sh \ -+ tests/i18n/sort-month.sh \ - tests/misc/sort-merge.pl \ - tests/misc/sort-merge-fdlimit.sh \ - tests/misc/sort-month.sh \ -diff -Naurp coreutils-8.25-orig/tests/misc/cut.pl coreutils-8.25/tests/misc/cut.pl ---- coreutils-8.25-orig/tests/misc/cut.pl 2016-01-16 12:18:13.000000000 -0600 -+++ coreutils-8.25/tests/misc/cut.pl 2016-02-08 19:07:10.314944661 -0600 -@@ -23,9 +23,11 @@ use strict; - # Turn off localization of executable's output. - @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; - --my $mb_locale = $ENV{LOCALE_FR_UTF8}; -+my $mb_locale; -+# uncommented enable multibyte paths -+$mb_locale = $ENV{LOCALE_FR_UTF8}; - ! defined $mb_locale || $mb_locale eq 'none' -- and $mb_locale = 'C'; -+ and $mb_locale = 'C'; - - my $prog = 'cut'; - my $try = "Try '$prog --help' for more information.\n"; -@@ -240,6 +242,7 @@ if ($mb_locale ne 'C') - my @new_t = @$t; - my $test_name = shift @new_t; - -+ next if ($test_name =~ "newline-[12][0-9]"); - push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; - } - push @Tests, @new; -diff -Naurp coreutils-8.25-orig/tests/misc/expand.pl coreutils-8.25/tests/misc/expand.pl ---- coreutils-8.25-orig/tests/misc/expand.pl 2016-01-16 12:18:13.000000000 -0600 -+++ coreutils-8.25/tests/misc/expand.pl 2016-02-08 19:07:10.314944661 -0600 -@@ -23,6 +23,15 @@ use strict; - # Turn off localization of executable's output. - @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; - -+#comment out next line to disable multibyte tests -+my $mb_locale = $ENV{LOCALE_FR_UTF8}; -+! defined $mb_locale || $mb_locale eq 'none' -+ and $mb_locale = 'C'; -+ -+my $prog = 'expand'; -+my $try = "Try \`$prog --help' for more information.\n"; -+my $inval = "$prog: invalid byte, character or field list\n$try"; -+ - my @Tests = - ( - ['t1', '--tabs=3', {IN=>"a\tb"}, {OUT=>"a b"}], -@@ -31,6 +40,37 @@ my @Tests = - ['i2', '--tabs=3 -i', {IN=>" \ta\tb"}, {OUT=>" a\tb"}], - ); - -+if ($mb_locale ne 'C') -+ { -+ # Duplicate each test vector, appending "-mb" to the test name and -+ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we -+ # provide coverage for the distro-added multi-byte code paths. -+ my @new; -+ foreach my $t (@Tests) -+ { -+ my @new_t = @$t; -+ my $test_name = shift @new_t; -+ -+ # Depending on whether expand is multi-byte-patched, -+ # it emits different diagnostics: -+ # non-MB: invalid byte or field list -+ # MB: invalid byte, character or field list -+ # Adjust the expected error output accordingly. -+ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} -+ (@new_t)) -+ { -+ my $sub = {ERR_SUBST => 's/, character//'}; -+ push @new_t, $sub; -+ push @$t, $sub; -+ } -+ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; -+ } -+ push @Tests, @new; -+ } -+ -+ -+@Tests = triple_test \@Tests; -+ - my $save_temps = $ENV{DEBUG}; - my $verbose = $ENV{VERBOSE}; - -diff -Naurp coreutils-8.25-orig/tests/misc/fold.pl coreutils-8.25/tests/misc/fold.pl ---- coreutils-8.25-orig/tests/misc/fold.pl 2016-01-16 12:18:13.000000000 -0600 -+++ coreutils-8.25/tests/misc/fold.pl 2016-02-08 19:07:10.314944661 -0600 -@@ -20,9 +20,18 @@ use strict; - - (my $program_name = $0) =~ s|.*/||; - -+my $prog = 'fold'; -+my $try = "Try \`$prog --help' for more information.\n"; -+my $inval = "$prog: invalid byte, character or field list\n$try"; -+ - # Turn off localization of executable's output. - @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; - -+# uncommented to enable multibyte paths -+my $mb_locale = $ENV{LOCALE_FR_UTF8}; -+! defined $mb_locale || $mb_locale eq 'none' -+ and $mb_locale = 'C'; -+ - my @Tests = - ( - ['s1', '-w2 -s', {IN=>"a\t"}, {OUT=>"a\n\t"}], -@@ -31,9 +40,48 @@ my @Tests = - ['s4', '-w4 -s', {IN=>"abc ef\n"}, {OUT=>"abc \nef\n"}], - ); - -+# Add _POSIX2_VERSION=199209 to the environment of each test -+# that uses an old-style option like +1. -+if ($mb_locale ne 'C') -+ { -+ # Duplicate each test vector, appending "-mb" to the test name and -+ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we -+ # provide coverage for the distro-added multi-byte code paths. -+ my @new; -+ foreach my $t (@Tests) -+ { -+ my @new_t = @$t; -+ my $test_name = shift @new_t; -+ -+ # Depending on whether fold is multi-byte-patched, -+ # it emits different diagnostics: -+ # non-MB: invalid byte or field list -+ # MB: invalid byte, character or field list -+ # Adjust the expected error output accordingly. -+ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} -+ (@new_t)) -+ { -+ my $sub = {ERR_SUBST => 's/, character//'}; -+ push @new_t, $sub; -+ push @$t, $sub; -+ } -+ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; -+ } -+ push @Tests, @new; -+ } -+ -+@Tests = triple_test \@Tests; -+ -+# Remember that triple_test creates from each test with exactly one "IN" -+# file two more tests (.p and .r suffix on name) corresponding to reading -+# input from a file and from a pipe. The pipe-reading test would fail -+# due to a race condition about 1 in 20 times. -+# Remove the IN_PIPE version of the "output-is-input" test above. -+# The others aren't susceptible because they have three inputs each. -+@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; -+ - my $save_temps = $ENV{DEBUG}; - my $verbose = $ENV{VERBOSE}; - --my $prog = 'fold'; - my $fail = run_tests ($program_name, $prog, \@Tests, $save_temps, $verbose); - exit $fail; -diff -Naurp coreutils-8.25-orig/tests/misc/join.pl coreutils-8.25/tests/misc/join.pl ---- coreutils-8.25-orig/tests/misc/join.pl 2016-01-16 12:18:13.000000000 -0600 -+++ coreutils-8.25/tests/misc/join.pl 2016-02-08 19:07:10.315944664 -0600 -@@ -25,6 +25,15 @@ my $limits = getlimits (); - - my $prog = 'join'; - -+my $try = "Try \`$prog --help' for more information.\n"; -+my $inval = "$prog: invalid byte, character or field list\n$try"; -+ -+my $mb_locale; -+#Comment out next line to disable multibyte tests -+$mb_locale = $ENV{LOCALE_FR_UTF8}; -+! defined $mb_locale || $mb_locale eq 'none' -+ and $mb_locale = 'C'; -+ - my $delim = chr 0247; - sub t_subst ($) - { -@@ -329,8 +338,49 @@ foreach my $t (@tv) - push @Tests, $new_ent; - } - -+# Add _POSIX2_VERSION=199209 to the environment of each test -+# that uses an old-style option like +1. -+if ($mb_locale ne 'C') -+ { -+ # Duplicate each test vector, appending "-mb" to the test name and -+ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we -+ # provide coverage for the distro-added multi-byte code paths. -+ my @new; -+ foreach my $t (@Tests) -+ { -+ my @new_t = @$t; -+ my $test_name = shift @new_t; -+ -+ # Depending on whether join is multi-byte-patched, -+ # it emits different diagnostics: -+ # non-MB: invalid byte or field list -+ # MB: invalid byte, character or field list -+ # Adjust the expected error output accordingly. -+ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} -+ (@new_t)) -+ { -+ my $sub = {ERR_SUBST => 's/, character//'}; -+ push @new_t, $sub; -+ push @$t, $sub; -+ } -+ #Adjust the output some error messages including test_name for mb -+ if (grep {ref $_ eq 'HASH' && exists $_->{ERR}} -+ (@new_t)) -+ { -+ my $sub2 = {ERR_SUBST => "s/$test_name-mb/$test_name/"}; -+ push @new_t, $sub2; -+ push @$t, $sub2; -+ } -+ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; -+ } -+ push @Tests, @new; -+ } -+ - @Tests = triple_test \@Tests; - -+#skip invalid-j-mb test, it is failing because of the format -+@Tests = grep {$_->[0] ne 'invalid-j-mb'} @Tests; -+ - my $save_temps = $ENV{DEBUG}; - my $verbose = $ENV{VERBOSE}; - -diff -Naurp coreutils-8.25-orig/tests/misc/sort-mb-tests.sh coreutils-8.25/tests/misc/sort-mb-tests.sh ---- coreutils-8.25-orig/tests/misc/sort-mb-tests.sh 1969-12-31 18:00:00.000000000 -0600 -+++ coreutils-8.25/tests/misc/sort-mb-tests.sh 2016-02-08 19:07:10.315944664 -0600 -@@ -0,0 +1,45 @@ -+#!/bin/sh -+# Verify sort's multi-byte support. -+ -+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src -+print_ver_ sort -+ -+export LC_ALL=en_US.UTF-8 -+locale -k LC_CTYPE | grep -q "charmap.*UTF-8" \ -+ || skip_ "No UTF-8 locale available" -+ -+ -+cat < exp -+Banana＠5 -+Apple＠10 -+Citrus＠20 -+Cherry＠30 -+EOF -+ -+cat < out || fail=1 -+Apple＠10 -+Banana＠5 -+Citrus＠20 -+Cherry＠30 -+EOF -+ -+compare exp out || { fail=1; cat out; } -+ -+ -+cat < exp -+Citrus＠ＡＡ20＠＠5 -+Cherry＠ＡＡ30＠＠10 -+Apple＠ＡＡ10＠＠20 -+Banana＠ＡＡ5＠＠30 -+EOF -+ -+cat < out || fail=1 -+Apple＠ＡＡ10＠＠20 -+Banana＠ＡＡ5＠＠30 -+Citrus＠ＡＡ20＠＠5 -+Cherry＠ＡＡ30＠＠10 -+EOF -+ -+compare exp out || { fail=1; cat out; } -+ -+Exit $fail -diff -Naurp coreutils-8.25-orig/tests/misc/sort-merge.pl coreutils-8.25/tests/misc/sort-merge.pl ---- coreutils-8.25-orig/tests/misc/sort-merge.pl 2016-01-16 12:18:14.000000000 -0600 -+++ coreutils-8.25/tests/misc/sort-merge.pl 2016-02-08 19:07:10.316944667 -0600 -@@ -26,6 +26,15 @@ my $prog = 'sort'; - # Turn off localization of executable's output. - @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; - -+my $mb_locale; -+# uncommented according to upstream commit enabling multibyte paths -+$mb_locale = $ENV{LOCALE_FR_UTF8}; -+! defined $mb_locale || $mb_locale eq 'none' -+ and $mb_locale = 'C'; -+ -+my $try = "Try \`$prog --help' for more information.\n"; -+my $inval = "$prog: invalid byte, character or field list\n$try"; -+ - # three empty files and one that says 'foo' - my @inputs = (+(map{{IN=> {"empty$_"=> ''}}}1..3), {IN=> {foo=> "foo\n"}}); - -@@ -77,6 +86,39 @@ my @Tests = - {OUT=>$big_input}], - ); - -+# Add _POSIX2_VERSION=199209 to the environment of each test -+# that uses an old-style option like +1. -+if ($mb_locale ne 'C') -+ { -+ # Duplicate each test vector, appending "-mb" to the test name and -+ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we -+ # provide coverage for the distro-added multi-byte code paths. -+ my @new; -+ foreach my $t (@Tests) -+ { -+ my @new_t = @$t; -+ my $test_name = shift @new_t; -+ -+ # Depending on whether sort is multi-byte-patched, -+ # it emits different diagnostics: -+ # non-MB: invalid byte or field list -+ # MB: invalid byte, character or field list -+ # Adjust the expected error output accordingly. -+ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} -+ (@new_t)) -+ { -+ my $sub = {ERR_SUBST => 's/, character//'}; -+ push @new_t, $sub; -+ push @$t, $sub; -+ } -+ next if ($test_name =~ "nmerge-."); -+ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; -+ } -+ push @Tests, @new; -+ } -+ -+@Tests = triple_test \@Tests; -+ - my $save_temps = $ENV{DEBUG}; - my $verbose = $ENV{VERBOSE}; - -diff -Naurp coreutils-8.25-orig/tests/misc/sort.pl coreutils-8.25/tests/misc/sort.pl ---- coreutils-8.25-orig/tests/misc/sort.pl 2016-01-16 12:18:14.000000000 -0600 -+++ coreutils-8.25/tests/misc/sort.pl 2016-02-08 19:07:10.316944667 -0600 -@@ -24,10 +24,15 @@ my $prog = 'sort'; - # Turn off localization of executable's output. - @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; - --my $mb_locale = $ENV{LOCALE_FR_UTF8}; -+my $mb_locale; -+#Comment out next line to disable multibyte tests -+$mb_locale = $ENV{LOCALE_FR_UTF8}; - ! defined $mb_locale || $mb_locale eq 'none' - and $mb_locale = 'C'; - -+my $try = "Try \`$prog --help' for more information.\n"; -+my $inval = "$prog: invalid byte, character or field list\n$try"; -+ - # Since each test is run with a file name and with redirected stdin, - # the name in the diagnostic is either the file name or "-". - # Normalize each diagnostic to use '-'. -@@ -424,6 +429,38 @@ foreach my $t (@Tests) - } - } - -+if ($mb_locale ne 'C') -+ { -+ # Duplicate each test vector, appending "-mb" to the test name and -+ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we -+ # provide coverage for the distro-added multi-byte code paths. -+ my @new; -+ foreach my $t (@Tests) -+ { -+ my @new_t = @$t; -+ my $test_name = shift @new_t; -+ -+ # Depending on whether sort is multi-byte-patched, -+ # it emits different diagnostics: -+ # non-MB: invalid byte or field list -+ # MB: invalid byte, character or field list -+ # Adjust the expected error output accordingly. -+ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} -+ (@new_t)) -+ { -+ my $sub = {ERR_SUBST => 's/, character//'}; -+ push @new_t, $sub; -+ push @$t, $sub; -+ } -+ #disable several failing tests until investigation, disable all tests with envvars set -+ next if (grep {ref $_ eq 'HASH' && exists $_->{ENV}} (@new_t)); -+ next if ($test_name =~ "18g" or $test_name =~ "sort-numeric" or $test_name =~ "08[ab]" or $test_name =~ "03[def]" or $test_name =~ "h4" or $test_name =~ "n1" or $test_name =~ "2[01]a"); -+ next if ($test_name =~ "11[ab]"); # avoid FP: expected result differs to MB result due to collation rules. -+ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; -+ } -+ push @Tests, @new; -+ } -+ - @Tests = triple_test \@Tests; - - # Remember that triple_test creates from each test with exactly one "IN" -@@ -433,6 +470,7 @@ foreach my $t (@Tests) - # Remove the IN_PIPE version of the "output-is-input" test above. - # The others aren't susceptible because they have three inputs each. - @Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; -+@Tests = grep {$_->[0] ne 'output-is-input-mb.p'} @Tests; - - my $save_temps = $ENV{DEBUG}; - my $verbose = $ENV{VERBOSE}; -diff -Naurp coreutils-8.25-orig/tests/misc/unexpand.pl coreutils-8.25/tests/misc/unexpand.pl ---- coreutils-8.25-orig/tests/misc/unexpand.pl 2016-01-16 12:18:14.000000000 -0600 -+++ coreutils-8.25/tests/misc/unexpand.pl 2016-02-08 19:07:10.317944671 -0600 -@@ -27,6 +27,14 @@ my $limits = getlimits (); - - my $prog = 'unexpand'; - -+# comment out next line to disable multibyte tests -+my $mb_locale = $ENV{LOCALE_FR_UTF8}; -+! defined $mb_locale || $mb_locale eq 'none' -+ and $mb_locale = 'C'; -+ -+my $try = "Try \`$prog --help' for more information.\n"; -+my $inval = "$prog: invalid byte, character or field list\n$try"; -+ - my @Tests = - ( - ['a1', {IN=> ' 'x 1 ."y\n"}, {OUT=> ' 'x 1 ."y\n"}], -@@ -92,6 +100,37 @@ my @Tests = - {EXIT => 1}, {ERR => "$prog: tab stop value is too large\n"}], - ); - -+if ($mb_locale ne 'C') -+ { -+ # Duplicate each test vector, appending "-mb" to the test name and -+ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we -+ # provide coverage for the distro-added multi-byte code paths. -+ my @new; -+ foreach my $t (@Tests) -+ { -+ my @new_t = @$t; -+ my $test_name = shift @new_t; -+ -+ # Depending on whether unexpand is multi-byte-patched, -+ # it emits different diagnostics: -+ # non-MB: invalid byte or field list -+ # MB: invalid byte, character or field list -+ # Adjust the expected error output accordingly. -+ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} -+ (@new_t)) -+ { -+ my $sub = {ERR_SUBST => 's/, character//'}; -+ push @new_t, $sub; -+ push @$t, $sub; -+ } -+ next if ($test_name =~ 'b-1'); -+ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; -+ } -+ push @Tests, @new; -+ } -+ -+@Tests = triple_test \@Tests; -+ - my $save_temps = $ENV{DEBUG}; - my $verbose = $ENV{VERBOSE}; - -diff -Naurp coreutils-8.25-orig/tests/misc/uniq.pl coreutils-8.25/tests/misc/uniq.pl ---- coreutils-8.25-orig/tests/misc/uniq.pl 2016-01-16 12:18:14.000000000 -0600 -+++ coreutils-8.25/tests/misc/uniq.pl 2016-02-08 19:07:10.317944671 -0600 -@@ -23,9 +23,17 @@ my $limits = getlimits (); - my $prog = 'uniq'; - my $try = "Try '$prog --help' for more information.\n"; - -+my $inval = "$prog: invalid byte, character or field list\n$try"; -+ - # Turn off localization of executable's output. - @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; - -+my $mb_locale; -+#Comment out next line to disable multibyte tests -+$mb_locale = $ENV{LOCALE_FR_UTF8}; -+! defined $mb_locale || $mb_locale eq 'none' -+ and $mb_locale = 'C'; -+ - # When possible, create a "-z"-testing variant of each test. - sub add_z_variants($) - { -@@ -262,6 +270,53 @@ foreach my $t (@Tests) - and push @$t, {ENV=>'_POSIX2_VERSION=199209'}; - } - -+if ($mb_locale ne 'C') -+ { -+ # Duplicate each test vector, appending "-mb" to the test name and -+ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we -+ # provide coverage for the distro-added multi-byte code paths. -+ my @new; -+ foreach my $t (@Tests) -+ { -+ my @new_t = @$t; -+ my $test_name = shift @new_t; -+ -+ # Depending on whether uniq is multi-byte-patched, -+ # it emits different diagnostics: -+ # non-MB: invalid byte or field list -+ # MB: invalid byte, character or field list -+ # Adjust the expected error output accordingly. -+ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} -+ (@new_t)) -+ { -+ my $sub = {ERR_SUBST => 's/, character//'}; -+ push @new_t, $sub; -+ push @$t, $sub; -+ } -+ # In test #145, replace the each ‘...’ by '...'. -+ if ($test_name =~ "145") -+ { -+ my $sub = { ERR_SUBST => "s/‘([^’]+)’/'\$1'/g"}; -+ push @new_t, $sub; -+ push @$t, $sub; -+ } -+ next if ( $test_name =~ "schar" -+ or $test_name =~ "^obs-plus" -+ or $test_name =~ "119"); -+ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; -+ } -+ push @Tests, @new; -+ } -+ -+# Remember that triple_test creates from each test with exactly one "IN" -+# file two more tests (.p and .r suffix on name) corresponding to reading -+# input from a file and from a pipe. The pipe-reading test would fail -+# due to a race condition about 1 in 20 times. -+# Remove the IN_PIPE version of the "output-is-input" test above. -+# The others aren't susceptible because they have three inputs each. -+ -+@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; -+ - @Tests = add_z_variants \@Tests; - @Tests = triple_test \@Tests; - -diff -Naurp coreutils-8.25-orig/tests/pr/pr-tests.pl coreutils-8.25/tests/pr/pr-tests.pl ---- coreutils-8.25-orig/tests/pr/pr-tests.pl 2016-01-16 12:18:14.000000000 -0600 -+++ coreutils-8.25/tests/pr/pr-tests.pl 2016-02-08 19:07:10.318944674 -0600 -@@ -24,6 +24,15 @@ use strict; - my $prog = 'pr'; - my $normalize_strerror = "s/': .*/'/"; - -+my $mb_locale; -+#Uncomment the following line to enable multibyte tests -+$mb_locale = $ENV{LOCALE_FR_UTF8}; -+! defined $mb_locale || $mb_locale eq 'none' -+ and $mb_locale = 'C'; -+ -+my $try = "Try \`$prog --help' for more information.\n"; -+my $inval = "$prog: invalid byte, character or field list\n$try"; -+ - my @tv = ( - - # -b option is no longer an official option. But it's still working to -@@ -467,8 +476,48 @@ push @Tests, - {IN=>{3=>"x\ty\tz\n"}}, - {OUT=>join("\t", qw(a b c m n o x y z)) . "\n"} ]; - -+# Add _POSIX2_VERSION=199209 to the environment of each test -+# that uses an old-style option like +1. -+if ($mb_locale ne 'C') -+ { -+ # Duplicate each test vector, appending "-mb" to the test name and -+ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we -+ # provide coverage for the distro-added multi-byte code paths. -+ my @new; -+ foreach my $t (@Tests) -+ { -+ my @new_t = @$t; -+ my $test_name = shift @new_t; -+ -+ # Depending on whether pr is multi-byte-patched, -+ # it emits different diagnostics: -+ # non-MB: invalid byte or field list -+ # MB: invalid byte, character or field list -+ # Adjust the expected error output accordingly. -+ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} -+ (@new_t)) -+ { -+ my $sub = {ERR_SUBST => 's/, character//'}; -+ push @new_t, $sub; -+ push @$t, $sub; -+ } -+ #temporarily skip some failing tests -+ next if ($test_name =~ "col-0" or $test_name =~ "col-inval"); -+ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; -+ } -+ push @Tests, @new; -+ } -+ - @Tests = triple_test \@Tests; - -+# Remember that triple_test creates from each test with exactly one "IN" -+# file two more tests (.p and .r suffix on name) corresponding to reading -+# input from a file and from a pipe. The pipe-reading test would fail -+# due to a race condition about 1 in 20 times. -+# Remove the IN_PIPE version of the "output-is-input" test above. -+# The others aren't susceptible because they have three inputs each. -+@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; -+ - my $save_temps = $ENV{DEBUG}; - my $verbose = $ENV{VERBOSE}; - diff -r c3c324c07d3f -r 90a5eb560fd6 coreutils/stuff/patches/coreutils-8.27-i18n-1.patch --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/coreutils/stuff/patches/coreutils-8.27-i18n-1.patch Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,5521 @@ +Submitted by: DJ Lucas (dj_AT_linuxfromscratch_DOT_org) +Date: 2017-03-12 +Initial Package Version: 8.27 +Upstream Status: Rejected +Origin: Based on Fedora's i18n patches at + http://pkgs.fedoraproject.org/cgit/rpms/coreutils.git/tree/ +Description: Fixes i18n issues with various Coreutils programs + +diff -Naurp coreutils-8.27-orig/bootstrap.conf coreutils-8.27/bootstrap.conf +--- coreutils-8.27-orig/bootstrap.conf 2017-03-07 23:34:06.000000000 -0600 ++++ coreutils-8.27/bootstrap.conf 2017-03-11 23:47:38.068058445 -0600 +@@ -152,6 +152,7 @@ gnulib_modules=" + maintainer-makefile + malloc-gnu + manywarnings ++ mbfile + mbrlen + mbrtowc + mbsalign +diff -Naurp coreutils-8.27-orig/configure.ac coreutils-8.27/configure.ac +--- coreutils-8.27-orig/configure.ac 2017-02-26 08:52:29.000000000 -0600 ++++ coreutils-8.27/configure.ac 2017-03-11 23:47:38.068058445 -0600 +@@ -429,6 +429,8 @@ fi + # I'm leaving it here for now. This whole thing needs to be modernized... + gl_WINSIZE_IN_PTEM + ++gl_MBFILE ++ + gl_HEADER_TIOCGWINSZ_IN_TERMIOS_H + + if test $gl_cv_sys_tiocgwinsz_needs_termios_h = no && \ +diff -Naurp coreutils-8.27-orig/lib/linebuffer.h coreutils-8.27/lib/linebuffer.h +--- coreutils-8.27-orig/lib/linebuffer.h 2017-01-01 16:35:38.000000000 -0600 ++++ coreutils-8.27/lib/linebuffer.h 2017-03-11 23:47:13.089286391 -0600 +@@ -21,6 +21,11 @@ + + # include + ++/* Get mbstate_t. */ ++# if HAVE_WCHAR_H ++# include ++# endif ++ + /* A 'struct linebuffer' holds a line of text. */ + + struct linebuffer +@@ -28,6 +33,9 @@ struct linebuffer + size_t size; /* Allocated. */ + size_t length; /* Used. */ + char *buffer; ++# if HAVE_WCHAR_H ++ mbstate_t state; ++# endif + }; + + /* Initialize linebuffer LINEBUFFER for use. */ +diff -Naurp coreutils-8.27-orig/lib/mbfile.c coreutils-8.27/lib/mbfile.c +--- coreutils-8.27-orig/lib/mbfile.c 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/lib/mbfile.c 2017-03-11 23:47:38.069058397 -0600 +@@ -0,0 +1,3 @@ ++#include ++#define MBFILE_INLINE _GL_EXTERN_INLINE ++#include "mbfile.h" +diff -Naurp coreutils-8.27-orig/lib/mbfile.h coreutils-8.27/lib/mbfile.h +--- coreutils-8.27-orig/lib/mbfile.h 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/lib/mbfile.h 2017-03-11 23:47:38.069058397 -0600 +@@ -0,0 +1,255 @@ ++/* Multibyte character I/O: macros for multi-byte encodings. ++ Copyright (C) 2001, 2005, 2009-2017 Free Software Foundation, Inc. ++ ++ This program is free software: you can redistribute it and/or modify ++ it under the terms of the GNU General Public License as published by ++ the Free Software Foundation; either version 3 of the License, or ++ (at your option) any later version. ++ ++ This program is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ GNU General Public License for more details. ++ ++ You should have received a copy of the GNU General Public License ++ along with this program. If not, see . */ ++ ++/* Written by Mitsuru Chinen ++ and Bruno Haible . */ ++ ++/* The macros in this file implement multi-byte character input from a ++ stream. ++ ++ mb_file_t ++ is the type for multibyte character input stream, usable for variable ++ declarations. ++ ++ mbf_char_t ++ is the type for multibyte character or EOF, usable for variable ++ declarations. ++ ++ mbf_init (mbf, stream) ++ initializes the MB_FILE for reading from stream. ++ ++ mbf_getc (mbc, mbf) ++ reads the next multibyte character from mbf and stores it in mbc. ++ ++ mb_iseof (mbc) ++ returns true if mbc represents the EOF value. ++ ++ Here are the function prototypes of the macros. ++ ++ extern void mbf_init (mb_file_t mbf, FILE *stream); ++ extern void mbf_getc (mbf_char_t mbc, mb_file_t mbf); ++ extern bool mb_iseof (const mbf_char_t mbc); ++ */ ++ ++#ifndef _MBFILE_H ++#define _MBFILE_H 1 ++ ++#include ++#include ++#include ++#include ++ ++/* Tru64 with Desktop Toolkit C has a bug: must be included before ++ . ++ BSD/OS 4.1 has a bug: and must be included before ++ . */ ++#include ++#include ++#include ++ ++#include "mbchar.h" ++ ++#ifndef _GL_INLINE_HEADER_BEGIN ++ #error "Please include config.h first." ++#endif ++_GL_INLINE_HEADER_BEGIN ++#ifndef MBFILE_INLINE ++# define MBFILE_INLINE _GL_INLINE ++#endif ++ ++struct mbfile_multi { ++ FILE *fp; ++ bool eof_seen; ++ bool have_pushback; ++ mbstate_t state; ++ unsigned int bufcount; ++ char buf[MBCHAR_BUF_SIZE]; ++ struct mbchar pushback; ++}; ++ ++MBFILE_INLINE void ++mbfile_multi_getc (struct mbchar *mbc, struct mbfile_multi *mbf) ++{ ++ size_t bytes; ++ ++ /* If EOF has already been seen, don't use getc. This matters if ++ mbf->fp is connected to an interactive tty. */ ++ if (mbf->eof_seen) ++ goto eof; ++ ++ /* Return character pushed back, if there is one. */ ++ if (mbf->have_pushback) ++ { ++ mb_copy (mbc, &mbf->pushback); ++ mbf->have_pushback = false; ++ return; ++ } ++ ++ /* Before using mbrtowc, we need at least one byte. */ ++ if (mbf->bufcount == 0) ++ { ++ int c = getc (mbf->fp); ++ if (c == EOF) ++ { ++ mbf->eof_seen = true; ++ goto eof; ++ } ++ mbf->buf[0] = (unsigned char) c; ++ mbf->bufcount++; ++ } ++ ++ /* Handle most ASCII characters quickly, without calling mbrtowc(). */ ++ if (mbf->bufcount == 1 && mbsinit (&mbf->state) && is_basic (mbf->buf[0])) ++ { ++ /* These characters are part of the basic character set. ISO C 99 ++ guarantees that their wide character code is identical to their ++ char code. */ ++ mbc->wc = mbc->buf[0] = mbf->buf[0]; ++ mbc->wc_valid = true; ++ mbc->ptr = &mbc->buf[0]; ++ mbc->bytes = 1; ++ mbf->bufcount = 0; ++ return; ++ } ++ ++ /* Use mbrtowc on an increasing number of bytes. Read only as many bytes ++ from mbf->fp as needed. This is needed to give reasonable interactive ++ behaviour when mbf->fp is connected to an interactive tty. */ ++ for (;;) ++ { ++ /* We don't know whether the 'mbrtowc' function updates the state when ++ it returns -2, - this is the ISO C 99 and glibc-2.2 behaviour - or ++ not - amended ANSI C, glibc-2.1 and Solaris 2.7 behaviour. We ++ don't have an autoconf test for this, yet. ++ The new behaviour would allow us to feed the bytes one by one into ++ mbrtowc. But the old behaviour forces us to feed all bytes since ++ the end of the last character into mbrtowc. Since we want to retry ++ with more bytes when mbrtowc returns -2, we must backup the state ++ before calling mbrtowc, because implementations with the new ++ behaviour will clobber it. */ ++ mbstate_t backup_state = mbf->state; ++ ++ bytes = mbrtowc (&mbc->wc, &mbf->buf[0], mbf->bufcount, &mbf->state); ++ ++ if (bytes == (size_t) -1) ++ { ++ /* An invalid multibyte sequence was encountered. */ ++ /* Return a single byte. */ ++ bytes = 1; ++ mbc->wc_valid = false; ++ break; ++ } ++ else if (bytes == (size_t) -2) ++ { ++ /* An incomplete multibyte character. */ ++ mbf->state = backup_state; ++ if (mbf->bufcount == MBCHAR_BUF_SIZE) ++ { ++ /* An overlong incomplete multibyte sequence was encountered. */ ++ /* Return a single byte. */ ++ bytes = 1; ++ mbc->wc_valid = false; ++ break; ++ } ++ else ++ { ++ /* Read one more byte and retry mbrtowc. */ ++ int c = getc (mbf->fp); ++ if (c == EOF) ++ { ++ /* An incomplete multibyte character at the end. */ ++ mbf->eof_seen = true; ++ bytes = mbf->bufcount; ++ mbc->wc_valid = false; ++ break; ++ } ++ mbf->buf[mbf->bufcount] = (unsigned char) c; ++ mbf->bufcount++; ++ } ++ } ++ else ++ { ++ if (bytes == 0) ++ { ++ /* A null wide character was encountered. */ ++ bytes = 1; ++ assert (mbf->buf[0] == '\0'); ++ assert (mbc->wc == 0); ++ } ++ mbc->wc_valid = true; ++ break; ++ } ++ } ++ ++ /* Return the multibyte sequence mbf->buf[0..bytes-1]. */ ++ mbc->ptr = &mbc->buf[0]; ++ memcpy (&mbc->buf[0], &mbf->buf[0], bytes); ++ mbc->bytes = bytes; ++ ++ mbf->bufcount -= bytes; ++ if (mbf->bufcount > 0) ++ { ++ /* It's not worth calling memmove() for so few bytes. */ ++ unsigned int count = mbf->bufcount; ++ char *p = &mbf->buf[0]; ++ ++ do ++ { ++ *p = *(p + bytes); ++ p++; ++ } ++ while (--count > 0); ++ } ++ return; ++ ++eof: ++ /* An mbchar_t with bytes == 0 is used to indicate EOF. */ ++ mbc->ptr = NULL; ++ mbc->bytes = 0; ++ mbc->wc_valid = false; ++ return; ++} ++ ++MBFILE_INLINE void ++mbfile_multi_ungetc (const struct mbchar *mbc, struct mbfile_multi *mbf) ++{ ++ mb_copy (&mbf->pushback, mbc); ++ mbf->have_pushback = true; ++} ++ ++typedef struct mbfile_multi mb_file_t; ++ ++typedef mbchar_t mbf_char_t; ++ ++#define mbf_init(mbf, stream) \ ++ ((mbf).fp = (stream), \ ++ (mbf).eof_seen = false, \ ++ (mbf).have_pushback = false, \ ++ memset (&(mbf).state, '\0', sizeof (mbstate_t)), \ ++ (mbf).bufcount = 0) ++ ++#define mbf_getc(mbc, mbf) mbfile_multi_getc (&(mbc), &(mbf)) ++ ++#define mbf_ungetc(mbc, mbf) mbfile_multi_ungetc (&(mbc), &(mbf)) ++ ++#define mb_iseof(mbc) ((mbc).bytes == 0) ++ ++#ifndef _GL_INLINE_HEADER_BEGIN ++ #error "Please include config.h first." ++#endif ++_GL_INLINE_HEADER_BEGIN ++ ++#endif /* _MBFILE_H */ +diff -Naurp coreutils-8.27-orig/m4/mbfile.m4 coreutils-8.27/m4/mbfile.m4 +--- coreutils-8.27-orig/m4/mbfile.m4 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/m4/mbfile.m4 2017-03-11 23:47:38.070058349 -0600 +@@ -0,0 +1,14 @@ ++# mbfile.m4 serial 7 ++dnl Copyright (C) 2005, 2008-2017 Free Software Foundation, Inc. ++dnl This file is free software; the Free Software Foundation ++dnl gives unlimited permission to copy and/or distribute it, ++dnl with or without modifications, as long as this notice is preserved. ++ ++dnl autoconf tests required for use of mbfile.h ++dnl From Bruno Haible. ++ ++AC_DEFUN([gl_MBFILE], ++[ ++ AC_REQUIRE([AC_TYPE_MBSTATE_T]) ++ : ++]) +diff -Naurp coreutils-8.27-orig/src/cut.c coreutils-8.27/src/cut.c +--- coreutils-8.27-orig/src/cut.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/cut.c 2017-03-11 23:47:59.526048471 -0600 +@@ -28,6 +28,11 @@ + #include + #include + #include ++ ++/* Get mbstate_t, mbrtowc(). */ ++#if HAVE_WCHAR_H ++# include ++#endif + #include "system.h" + + #include "error.h" +@@ -38,6 +43,18 @@ + + #include "set-fields.h" + ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 ++# undef MB_LEN_MAX ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ + /* The official name of this program (e.g., no 'g' prefix). */ + #define PROGRAM_NAME "cut" + +@@ -54,6 +71,52 @@ + } \ + while (0) + ++/* Refill the buffer BUF to get a multibyte character. */ ++#define REFILL_BUFFER(BUF, BUFPOS, BUFLEN, STREAM) \ ++ do \ ++ { \ ++ if (BUFLEN < MB_LEN_MAX && !feof (STREAM) && !ferror (STREAM)) \ ++ { \ ++ memmove (BUF, BUFPOS, BUFLEN); \ ++ BUFLEN += fread (BUF + BUFLEN, sizeof(char), BUFSIZ, STREAM); \ ++ BUFPOS = BUF; \ ++ } \ ++ } \ ++ while (0) ++ ++/* Get wide character on BUFPOS. BUFPOS is not included after that. ++ If byte sequence is not valid as a character, CONVFAIL is true. Otherwise false. */ ++#define GET_NEXT_WC_FROM_BUFFER(WC, BUFPOS, BUFLEN, MBLENGTH, STATE, CONVFAIL) \ ++ do \ ++ { \ ++ mbstate_t state_bak; \ ++ \ ++ if (BUFLEN < 1) \ ++ { \ ++ WC = WEOF; \ ++ break; \ ++ } \ ++ \ ++ /* Get a wide character. */ \ ++ CONVFAIL = false; \ ++ state_bak = STATE; \ ++ MBLENGTH = mbrtowc ((wchar_t *)&WC, BUFPOS, BUFLEN, &STATE); \ ++ \ ++ switch (MBLENGTH) \ ++ { \ ++ case (size_t)-1: \ ++ case (size_t)-2: \ ++ CONVFAIL = true; \ ++ STATE = state_bak; \ ++ /* Fall througn. */ \ ++ \ ++ case 0: \ ++ MBLENGTH = 1; \ ++ break; \ ++ } \ ++ } \ ++ while (0) ++ + + /* Pointer inside RP. When checking if a byte or field is selected + by a finite range, we check if it is between CURRENT_RP.LO +@@ -61,6 +124,9 @@ + CURRENT_RP.HI then we make CURRENT_RP to point to the next range pair. */ + static struct field_range_pair *current_rp; + ++/* Length of the delimiter given as argument to -d. */ ++size_t delimlen; ++ + /* This buffer is used to support the semantics of the -s option + (or lack of same) when the specified field list includes (does + not include) the first field. In both of those cases, the entire +@@ -77,15 +143,25 @@ enum operating_mode + { + undefined_mode, + +- /* Output characters that are in the given bytes. */ ++ /* Output bytes that are at the given positions. */ + byte_mode, + ++ /* Output characters that are at the given positions. */ ++ character_mode, ++ + /* Output the given delimiter-separated fields. */ + field_mode + }; + + static enum operating_mode operating_mode; + ++/* If nonzero, when in byte mode, don't split multibyte characters. */ ++static int byte_mode_character_aware; ++ ++/* If nonzero, the function for single byte locale is work ++ if this program runs on multibyte locale. */ ++static int force_singlebyte_mode; ++ + /* If true do not output lines containing no delimiter characters. + Otherwise, all such lines are printed. This option is valid only + with field mode. */ +@@ -97,6 +173,9 @@ static bool complement; + + /* The delimiter character for field mode. */ + static unsigned char delim; ++#if HAVE_WCHAR_H ++static wchar_t wcdelim; ++#endif + + /* The delimiter for each line/record. */ + static unsigned char line_delim = '\n'; +@@ -164,7 +243,7 @@ Print selected parts of lines from each + -f, --fields=LIST select only these fields; also print any line\n\ + that contains no delimiter character, unless\n\ + the -s option is specified\n\ +- -n (ignored)\n\ ++ -n with -b: don't split multibyte characters\n\ + "), stdout); + fputs (_("\ + --complement complement the set of selected bytes, characters\n\ +@@ -280,6 +359,82 @@ cut_bytes (FILE *stream) + } + } + ++#if HAVE_MBRTOWC ++/* This function is in use for the following case. ++ ++ 1. Read from the stream STREAM, printing to standard output any selected ++ characters. ++ ++ 2. Read from stream STREAM, printing to standard output any selected bytes, ++ without splitting multibyte characters. */ ++ ++static void ++cut_characters_or_cut_bytes_no_split (FILE *stream) ++{ ++ size_t idx; /* number of bytes or characters in the line so far. */ ++ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ ++ char *bufpos; /* Next read position of BUF. */ ++ size_t buflen; /* The length of the byte sequence in buf. */ ++ wint_t wc; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character which shows ++ as same character as WC. */ ++ mbstate_t state; /* State of the stream. */ ++ bool convfail = false; /* true, when conversion failed. Otherwise false. */ ++ /* Whether to begin printing delimiters between ranges for the current line. ++ Set after we've begun printing data corresponding to the first range. */ ++ bool print_delimiter = false; ++ ++ idx = 0; ++ buflen = 0; ++ bufpos = buf; ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ current_rp = frp; ++ ++ while (1) ++ { ++ REFILL_BUFFER (buf, bufpos, buflen, stream); ++ ++ GET_NEXT_WC_FROM_BUFFER (wc, bufpos, buflen, mblength, state, convfail); ++ (void) convfail; /* ignore unused */ ++ ++ if (wc == WEOF) ++ { ++ if (idx > 0) ++ putchar (line_delim); ++ break; ++ } ++ else if (wc == line_delim) ++ { ++ putchar (line_delim); ++ idx = 0; ++ print_delimiter = false; ++ current_rp = frp; ++ } ++ else ++ { ++ next_item (&idx); ++ if (print_kth (idx)) ++ { ++ if (output_delimiter_specified) ++ { ++ if (print_delimiter && is_range_start_index (idx)) ++ { ++ fwrite (output_delimiter_string, sizeof (char), ++ output_delimiter_length, stdout); ++ } ++ print_delimiter = true; ++ } ++ fwrite (bufpos, mblength, sizeof(char), stdout); ++ } ++ } ++ ++ buflen -= mblength; ++ bufpos += mblength; ++ } ++} ++#endif ++ + /* Read from stream STREAM, printing to standard output any selected fields. */ + + static void +@@ -425,13 +580,211 @@ cut_fields (FILE *stream) + } + } + ++#if HAVE_MBRTOWC ++static void ++cut_fields_mb (FILE *stream) ++{ ++ int c; ++ size_t field_idx; ++ int found_any_selected_field; ++ int buffer_first_field; ++ int empty_input; ++ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ ++ char *bufpos; /* Next read position of BUF. */ ++ size_t buflen; /* The length of the byte sequence in buf. */ ++ wint_t wc = 0; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character which shows ++ as same character as WC. */ ++ mbstate_t state; /* State of the stream. */ ++ bool convfail = false; /* true, when conversion failed. Otherwise false. */ ++ ++ current_rp = frp; ++ ++ found_any_selected_field = 0; ++ field_idx = 1; ++ bufpos = buf; ++ buflen = 0; ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ c = getc (stream); ++ empty_input = (c == EOF); ++ if (c != EOF) ++ { ++ ungetc (c, stream); ++ wc = 0; ++ } ++ else ++ wc = WEOF; ++ ++ /* To support the semantics of the -s flag, we may have to buffer ++ all of the first field to determine whether it is `delimited.' ++ But that is unnecessary if all non-delimited lines must be printed ++ and the first field has been selected, or if non-delimited lines ++ must be suppressed and the first field has *not* been selected. ++ That is because a non-delimited line has exactly one field. */ ++ buffer_first_field = (suppress_non_delimited ^ !print_kth (1)); ++ ++ while (1) ++ { ++ if (field_idx == 1 && buffer_first_field) ++ { ++ int len = 0; ++ ++ while (1) ++ { ++ REFILL_BUFFER (buf, bufpos, buflen, stream); ++ ++ GET_NEXT_WC_FROM_BUFFER ++ (wc, bufpos, buflen, mblength, state, convfail); ++ ++ if (wc == WEOF) ++ break; ++ ++ field_1_buffer = xrealloc (field_1_buffer, len + mblength); ++ memcpy (field_1_buffer + len, bufpos, mblength); ++ len += mblength; ++ buflen -= mblength; ++ bufpos += mblength; ++ ++ if (!convfail && (wc == line_delim || wc == wcdelim)) ++ break; ++ } ++ ++ if (len <= 0 && wc == WEOF) ++ break; ++ ++ /* If the first field extends to the end of line (it is not ++ delimited) and we are printing all non-delimited lines, ++ print this one. */ ++ if (convfail || (!convfail && wc != wcdelim)) ++ { ++ if (suppress_non_delimited) ++ { ++ /* Empty. */ ++ } ++ else ++ { ++ fwrite (field_1_buffer, sizeof (char), len, stdout); ++ /* Make sure the output line is newline terminated. */ ++ if (convfail || (!convfail && wc != line_delim)) ++ putchar (line_delim); ++ } ++ continue; ++ } ++ ++ if (print_kth (1)) ++ { ++ /* Print the field, but not the trailing delimiter. */ ++ fwrite (field_1_buffer, sizeof (char), len - 1, stdout); ++ found_any_selected_field = 1; ++ } ++ next_item (&field_idx); ++ } ++ ++ if (wc != WEOF) ++ { ++ if (print_kth (field_idx)) ++ { ++ if (found_any_selected_field) ++ { ++ fwrite (output_delimiter_string, sizeof (char), ++ output_delimiter_length, stdout); ++ } ++ found_any_selected_field = 1; ++ } ++ ++ while (1) ++ { ++ REFILL_BUFFER (buf, bufpos, buflen, stream); ++ ++ GET_NEXT_WC_FROM_BUFFER ++ (wc, bufpos, buflen, mblength, state, convfail); ++ ++ if (wc == WEOF) ++ break; ++ else if (!convfail && (wc == wcdelim || wc == line_delim)) ++ { ++ buflen -= mblength; ++ bufpos += mblength; ++ break; ++ } ++ ++ if (print_kth (field_idx)) ++ fwrite (bufpos, mblength, sizeof(char), stdout); ++ ++ buflen -= mblength; ++ bufpos += mblength; ++ } ++ } ++ ++ if ((!convfail || wc == line_delim) && buflen < 1) ++ wc = WEOF; ++ ++ if (!convfail && wc == wcdelim) ++ next_item (&field_idx); ++ else if (wc == WEOF || (!convfail && wc == line_delim)) ++ { ++ if (found_any_selected_field ++ || (!empty_input && !(suppress_non_delimited && field_idx == 1))) ++ putchar (line_delim); ++ if (wc == WEOF) ++ break; ++ field_idx = 1; ++ current_rp = frp; ++ found_any_selected_field = 0; ++ } ++ } ++} ++#endif ++ + static void + cut_stream (FILE *stream) + { +- if (operating_mode == byte_mode) +- cut_bytes (stream); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1 && !force_singlebyte_mode) ++ { ++ switch (operating_mode) ++ { ++ case byte_mode: ++ if (byte_mode_character_aware) ++ cut_characters_or_cut_bytes_no_split (stream); ++ else ++ cut_bytes (stream); ++ break; ++ ++ case character_mode: ++ cut_characters_or_cut_bytes_no_split (stream); ++ break; ++ ++ case field_mode: ++ if (delimlen == 1) ++ { ++ /* Check if we have utf8 multibyte locale, so we can use this ++ optimization because of uniqueness of characters, which is ++ not true for e.g. SJIS */ ++ char * loc = setlocale(LC_CTYPE, NULL); ++ if (loc && (strstr (loc, "UTF-8") || strstr (loc, "utf-8") || ++ strstr (loc, "UTF8") || strstr (loc, "utf8"))) ++ { ++ cut_fields (stream); ++ break; ++ } ++ } ++ cut_fields_mb (stream); ++ break; ++ ++ default: ++ abort (); ++ } ++ } + else +- cut_fields (stream); ++#endif ++ { ++ if (operating_mode == field_mode) ++ cut_fields (stream); ++ else ++ cut_bytes (stream); ++ } + } + + /* Process file FILE to standard output. +@@ -483,6 +836,7 @@ main (int argc, char **argv) + bool ok; + bool delim_specified = false; + char *spec_list_string IF_LINT ( = NULL); ++ char mbdelim[MB_LEN_MAX + 1]; + + initialize_main (&argc, &argv); + set_program_name (argv[0]); +@@ -505,7 +859,6 @@ main (int argc, char **argv) + switch (optc) + { + case 'b': +- case 'c': + /* Build the byte list. */ + if (operating_mode != undefined_mode) + FATAL_ERROR (_("only one type of list may be specified")); +@@ -513,6 +866,14 @@ main (int argc, char **argv) + spec_list_string = optarg; + break; + ++ case 'c': ++ /* Build the character list. */ ++ if (operating_mode != undefined_mode) ++ FATAL_ERROR (_("only one type of list may be specified")); ++ operating_mode = character_mode; ++ spec_list_string = optarg; ++ break; ++ + case 'f': + /* Build the field list. */ + if (operating_mode != undefined_mode) +@@ -524,10 +885,38 @@ main (int argc, char **argv) + case 'd': + /* New delimiter. */ + /* Interpret -d '' to mean 'use the NUL byte as the delimiter.' */ +- if (optarg[0] != '\0' && optarg[1] != '\0') +- FATAL_ERROR (_("the delimiter must be a single character")); +- delim = optarg[0]; +- delim_specified = true; ++ { ++#if HAVE_MBRTOWC ++ if(MB_CUR_MAX > 1) ++ { ++ mbstate_t state; ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ delimlen = mbrtowc (&wcdelim, optarg, strnlen(optarg, MB_LEN_MAX), &state); ++ ++ if (delimlen == (size_t)-1 || delimlen == (size_t)-2) ++ ++force_singlebyte_mode; ++ else ++ { ++ delimlen = (delimlen < 1) ? 1 : delimlen; ++ if (wcdelim != L'\0' && *(optarg + delimlen) != '\0') ++ FATAL_ERROR (_("the delimiter must be a single character")); ++ memcpy (mbdelim, optarg, delimlen); ++ mbdelim[delimlen] = '\0'; ++ if (delimlen == 1) ++ delim = *optarg; ++ } ++ } ++ ++ if (MB_CUR_MAX <= 1 || force_singlebyte_mode) ++#endif ++ { ++ if (optarg[0] != '\0' && optarg[1] != '\0') ++ FATAL_ERROR (_("the delimiter must be a single character")); ++ delim = (unsigned char) optarg[0]; ++ } ++ delim_specified = true; ++ } + break; + + case OUTPUT_DELIMITER_OPTION: +@@ -540,6 +929,7 @@ main (int argc, char **argv) + break; + + case 'n': ++ byte_mode_character_aware = 1; + break; + + case 's': +@@ -579,15 +969,34 @@ main (int argc, char **argv) + | (complement ? SETFLD_COMPLEMENT : 0) ); + + if (!delim_specified) +- delim = '\t'; ++ { ++ delim = '\t'; ++#ifdef HAVE_MBRTOWC ++ wcdelim = L'\t'; ++ mbdelim[0] = '\t'; ++ mbdelim[1] = '\0'; ++ delimlen = 1; ++#endif ++ } + + if (output_delimiter_string == NULL) + { +- static char dummy[2]; +- dummy[0] = delim; +- dummy[1] = '\0'; +- output_delimiter_string = dummy; +- output_delimiter_length = 1; ++#ifdef HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1 && !force_singlebyte_mode) ++ { ++ output_delimiter_string = xstrdup(mbdelim); ++ output_delimiter_length = delimlen; ++ } ++ ++ if (MB_CUR_MAX <= 1 || force_singlebyte_mode) ++#endif ++ { ++ static char dummy[2]; ++ dummy[0] = delim; ++ dummy[1] = '\0'; ++ output_delimiter_string = dummy; ++ output_delimiter_length = 1; ++ } + } + + if (optind == argc) +diff -Naurp coreutils-8.27-orig/src/expand.c coreutils-8.27/src/expand.c +--- coreutils-8.27-orig/src/expand.c 2017-02-26 15:42:25.000000000 -0600 ++++ coreutils-8.27/src/expand.c 2017-03-11 23:49:06.758133530 -0600 +@@ -37,6 +37,9 @@ + #include + #include + #include ++ ++#include ++ + #include "system.h" + #include "die.h" + #include "xstrndup.h" +@@ -100,19 +103,41 @@ expand (void) + { + /* Input stream. */ + FILE *fp = next_file (NULL); ++ mb_file_t mbf; ++ mbf_char_t c; ++ /* True if the starting locale is utf8. */ ++ bool using_utf_locale; ++ ++ /* True if the first file contains BOM header. */ ++ bool found_bom; ++ using_utf_locale=check_utf_locale(); + + if (!fp) + return; ++ mbf_init (mbf, fp); ++ found_bom=check_bom(fp,&mbf); + +- while (true) ++ if (using_utf_locale == false && found_bom == true) ++ { ++ /*try using some predefined locale */ ++ ++ if (set_utf_locale () != 0) + { +- /* Input character, or EOF. */ +- int c; ++ error (EXIT_FAILURE, errno, _("cannot set UTF-8 locale")); ++ } ++ } ++ + ++ if (found_bom == true) ++ { ++ print_bom(); ++ } ++ ++ while (true) ++ { + /* If true, perform translations. */ + bool convert = true; + +- + /* The following variables have valid values only when CONVERT + is true: */ + +@@ -122,17 +147,48 @@ expand (void) + /* Index in TAB_LIST of next tab stop to examine. */ + size_t tab_index = 0; + +- + /* Convert a line of text. */ + + do + { +- while ((c = getc (fp)) < 0 && (fp = next_file (fp))) +- continue; ++ while (true) { ++ mbf_getc (c, mbf); ++ if ((mb_iseof (c)) && (fp = next_file (fp))) ++ { ++ mbf_init (mbf, fp); ++ if (fp!=NULL) ++ { ++ if (check_bom(fp,&mbf)==true) ++ { ++ /*Not the first file - check BOM header*/ ++ if (using_utf_locale==false && found_bom==false) ++ { ++ /*BOM header in subsequent file but not in the first one. */ ++ error (EXIT_FAILURE, errno, _("combination of files with and without BOM header")); ++ } ++ } ++ else ++ { ++ if(using_utf_locale==false && found_bom==true) ++ { ++ /*First file conatined BOM header - locale was switched to UTF ++ /*all subsequent files should contain BOM. */ ++ error (EXIT_FAILURE, errno, _("combination of files with and without BOM header")); ++ } ++ } ++ } ++ continue; ++ } ++ else ++ { ++ break; ++ } ++ } ++ + + if (convert) + { +- if (c == '\t') ++ if (mb_iseq (c, '\t')) + { + /* Column the next input tab stop is on. */ + uintmax_t next_tab_column; +@@ -151,32 +207,34 @@ expand (void) + if (putchar (' ') < 0) + die (EXIT_FAILURE, errno, _("write error")); + +- c = ' '; ++ mb_setascii (&c, ' '); + } +- else if (c == '\b') ++ else if (mb_iseq (c, '\b')) + { + /* Go back one column, and force recalculation of the + next tab stop. */ + column -= !!column; + tab_index -= !!tab_index; + } +- else ++ /* A leading control character could make us trip over. */ ++ else if (!mb_iscntrl (c)) + { +- column++; ++ column += mb_width (c); + if (!column) + die (EXIT_FAILURE, 0, _("input line is too long")); + } + +- convert &= convert_entire_line || !! isblank (c); ++ convert &= convert_entire_line || mb_isblank (c); + } + +- if (c < 0) ++ if (mb_iseof (c)) + return; + +- if (putchar (c) < 0) ++ mb_putc (c, stdout); ++ if (ferror (stdout)) + die (EXIT_FAILURE, errno, _("write error")); + } +- while (c != '\n'); ++ while (!mb_iseq (c, '\n')); + } + } + +diff -Naurp coreutils-8.27-orig/src/expand-common.c coreutils-8.27/src/expand-common.c +--- coreutils-8.27-orig/src/expand-common.c 2017-03-01 11:22:55.000000000 -0600 ++++ coreutils-8.27/src/expand-common.c 2017-03-11 23:49:06.757133570 -0600 +@@ -18,6 +18,7 @@ + + #include + #include ++#include + #include "system.h" + #include "die.h" + #include "error.h" +@@ -105,6 +106,119 @@ set_extend_size (uintmax_t tabval) + return ok; + } + ++extern int ++set_utf_locale (void) ++{ ++ /*try using some predefined locale */ ++ const char* predef_locales[] = {"C.UTF8","en_US.UTF8","en_GB.UTF8"}; ++ ++ const int predef_locales_count=3; ++ for (int i=0;ibufcount=0; ++ if (c == 0xEF) ++ { ++ c=fgetc(fp); ++ } ++ else ++ { ++ if (c != EOF) ++ { ++ ungetc(c,fp); ++ } ++ return false; ++ } ++ ++ if (c == 0xBB) ++ { ++ c=fgetc(fp); ++ } ++ else ++ { ++ if ( c!= EOF ) ++ { ++ mbf->buf[0]=(unsigned char) 0xEF; ++ mbf->bufcount=1; ++ ungetc(c,fp); ++ return false; ++ } ++ else ++ { ++ ungetc(0xEF,fp); ++ return false; ++ } ++ } ++ if (c == 0xBF) ++ { ++ mbf->bufcount=0; ++ return true; ++ } ++ else ++ { ++ if (c != EOF) ++ { ++ mbf->buf[0]=(unsigned char) 0xEF; ++ mbf->buf[1]=(unsigned char) 0xBB; ++ mbf->bufcount=2; ++ ungetc(c,fp); ++ return false; ++ } ++ else ++ { ++ mbf->buf[0]=(unsigned char) 0xEF; ++ mbf->bufcount=1; ++ ungetc(0xBB,fp); ++ return false; ++ } ++ } ++ return false; ++} ++ ++extern void ++print_bom(void) ++{ ++ putc (0xEF, stdout); ++ putc (0xBB, stdout); ++ putc (0xBF, stdout); ++} ++ + /* Add the comma or blank separated list of tab stops STOPS + to the list of tab stops. */ + extern void +diff -Naurp coreutils-8.27-orig/src/expand-common.h coreutils-8.27/src/expand-common.h +--- coreutils-8.27-orig/src/expand-common.h 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/expand-common.h 2017-03-11 23:49:06.758133530 -0600 +@@ -34,6 +34,18 @@ extern size_t max_column_width; + /* The desired exit status. */ + extern int exit_status; + ++extern int ++set_utf_locale (void); ++ ++extern bool ++check_utf_locale(void); ++ ++extern bool ++check_bom(FILE* fp, mb_file_t *mbf); ++ ++extern void ++print_bom(void); ++ + /* Add tab stop TABVAL to the end of 'tab_list'. */ + extern void + add_tab_stop (uintmax_t tabval); +diff -Naurp coreutils-8.27-orig/src/fold.c coreutils-8.27/src/fold.c +--- coreutils-8.27-orig/src/fold.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/fold.c 2017-03-11 23:49:30.982169404 -0600 +@@ -22,12 +22,34 @@ + #include + #include + ++/* Get mbstate_t, mbrtowc(), wcwidth(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ ++/* Get iswprint(), iswblank(), wcwidth(). */ ++#if HAVE_WCTYPE_H ++# include ++#endif ++ + #include "system.h" + #include "die.h" + #include "error.h" + #include "fadvise.h" + #include "xdectoint.h" + ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 ++# undef MB_LEN_MAX ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ + #define TAB_WIDTH 8 + + /* The official name of this program (e.g., no 'g' prefix). */ +@@ -35,20 +57,41 @@ + + #define AUTHORS proper_name ("David MacKenzie") + ++#define FATAL_ERROR(Message) \ ++ do \ ++ { \ ++ error (0, 0, (Message)); \ ++ usage (2); \ ++ } \ ++ while (0) ++ ++enum operating_mode ++{ ++ /* Fold texts by columns that are at the given positions. */ ++ column_mode, ++ ++ /* Fold texts by bytes that are at the given positions. */ ++ byte_mode, ++ ++ /* Fold texts by characters that are at the given positions. */ ++ character_mode, ++}; ++ ++/* The argument shows current mode. (Default: column_mode) */ ++static enum operating_mode operating_mode; ++ + /* If nonzero, try to break on whitespace. */ + static bool break_spaces; + +-/* If nonzero, count bytes, not column positions. */ +-static bool count_bytes; +- + /* If nonzero, at least one of the files we read was standard input. */ + static bool have_read_stdin; + +-static char const shortopts[] = "bsw:0::1::2::3::4::5::6::7::8::9::"; ++static char const shortopts[] = "bcsw:0::1::2::3::4::5::6::7::8::9::"; + + static struct option const longopts[] = + { + {"bytes", no_argument, NULL, 'b'}, ++ {"characters", no_argument, NULL, 'c'}, + {"spaces", no_argument, NULL, 's'}, + {"width", required_argument, NULL, 'w'}, + {GETOPT_HELP_OPTION_DECL}, +@@ -76,6 +119,7 @@ Wrap input lines in each FILE, writing t + + fputs (_("\ + -b, --bytes count bytes rather than columns\n\ ++ -c, --characters count characters rather than columns\n\ + -s, --spaces break at spaces\n\ + -w, --width=WIDTH use WIDTH columns instead of 80\n\ + "), stdout); +@@ -93,7 +137,7 @@ Wrap input lines in each FILE, writing t + static size_t + adjust_column (size_t column, char c) + { +- if (!count_bytes) ++ if (operating_mode != byte_mode) + { + if (c == '\b') + { +@@ -116,30 +160,14 @@ adjust_column (size_t column, char c) + to stdout, with maximum line length WIDTH. + Return true if successful. */ + +-static bool +-fold_file (char const *filename, size_t width) ++static void ++fold_text (FILE *istream, size_t width, int *saved_errno) + { +- FILE *istream; + int c; + size_t column = 0; /* Screen column where next char will go. */ + size_t offset_out = 0; /* Index in 'line_out' for next char. */ + static char *line_out = NULL; + static size_t allocated_out = 0; +- int saved_errno; +- +- if (STREQ (filename, "-")) +- { +- istream = stdin; +- have_read_stdin = true; +- } +- else +- istream = fopen (filename, "r"); +- +- if (istream == NULL) +- { +- error (0, errno, "%s", quotef (filename)); +- return false; +- } + + fadvise (istream, FADVISE_SEQUENTIAL); + +@@ -169,6 +197,15 @@ fold_file (char const *filename, size_t + bool found_blank = false; + size_t logical_end = offset_out; + ++ /* If LINE_OUT has no wide character, ++ put a new wide character in LINE_OUT ++ if column is bigger than width. */ ++ if (offset_out == 0) ++ { ++ line_out[offset_out++] = c; ++ continue; ++ } ++ + /* Look for the last blank. */ + while (logical_end) + { +@@ -215,11 +252,220 @@ fold_file (char const *filename, size_t + line_out[offset_out++] = c; + } + +- saved_errno = errno; ++ *saved_errno = errno; ++ ++ if (offset_out) ++ fwrite (line_out, sizeof (char), (size_t) offset_out, stdout); ++ ++} ++ ++#if HAVE_MBRTOWC ++static void ++fold_multibyte_text (FILE *istream, size_t width, int *saved_errno) ++{ ++ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ ++ size_t buflen = 0; /* The length of the byte sequence in buf. */ ++ char *bufpos = buf; /* Next read position of BUF. */ ++ wint_t wc; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character which shows ++ as same character as WC. */ ++ mbstate_t state, state_bak; /* State of the stream. */ ++ int convfail = 0; /* 1, when conversion is failed. Otherwise 0. */ ++ ++ static char *line_out = NULL; ++ size_t offset_out = 0; /* Index in `line_out' for next char. */ ++ static size_t allocated_out = 0; ++ ++ int increment; ++ size_t column = 0; ++ ++ size_t last_blank_pos; ++ size_t last_blank_column; ++ int is_blank_seen; ++ int last_blank_increment = 0; ++ int is_bs_following_last_blank; ++ size_t bs_following_last_blank_num; ++ int is_cr_after_last_blank; ++ ++#define CLEAR_FLAGS \ ++ do \ ++ { \ ++ last_blank_pos = 0; \ ++ last_blank_column = 0; \ ++ is_blank_seen = 0; \ ++ is_bs_following_last_blank = 0; \ ++ bs_following_last_blank_num = 0; \ ++ is_cr_after_last_blank = 0; \ ++ } \ ++ while (0) ++ ++#define START_NEW_LINE \ ++ do \ ++ { \ ++ putchar ('\n'); \ ++ column = 0; \ ++ offset_out = 0; \ ++ CLEAR_FLAGS; \ ++ } \ ++ while (0) ++ ++ CLEAR_FLAGS; ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ for (;; bufpos += mblength, buflen -= mblength) ++ { ++ if (buflen < MB_LEN_MAX && !feof (istream) && !ferror (istream)) ++ { ++ memmove (buf, bufpos, buflen); ++ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, istream); ++ bufpos = buf; ++ } ++ ++ if (buflen < 1) ++ break; ++ ++ /* Get a wide character. */ ++ state_bak = state; ++ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &state); ++ ++ switch (mblength) ++ { ++ case (size_t)-1: ++ case (size_t)-2: ++ convfail++; ++ state = state_bak; ++ /* Fall through. */ ++ ++ case 0: ++ mblength = 1; ++ break; ++ } ++ ++rescan: ++ if (convfail) ++ increment = 1; ++ else if (wc == L'\n') ++ { ++ /* preserve newline */ ++ fwrite (line_out, sizeof(char), offset_out, stdout); ++ START_NEW_LINE; ++ continue; ++ } ++ else if (operating_mode == byte_mode) /* byte mode */ ++ increment = mblength; ++ else if (operating_mode == character_mode) /* character mode */ ++ increment = 1; ++ else /* column mode */ ++ { ++ switch (wc) ++ { ++ case L'\b': ++ increment = (column > 0) ? -1 : 0; ++ break; ++ ++ case L'\r': ++ increment = -1 * column; ++ break; ++ ++ case L'\t': ++ increment = 8 - column % 8; ++ break; ++ ++ default: ++ increment = wcwidth (wc); ++ increment = (increment < 0) ? 0 : increment; ++ } ++ } ++ ++ if (column + increment > width && break_spaces && last_blank_pos) ++ { ++ fwrite (line_out, sizeof(char), last_blank_pos, stdout); ++ putchar ('\n'); ++ ++ offset_out = offset_out - last_blank_pos; ++ column = column - last_blank_column + ((is_cr_after_last_blank) ++ ? last_blank_increment : bs_following_last_blank_num); ++ memmove (line_out, line_out + last_blank_pos, offset_out); ++ CLEAR_FLAGS; ++ goto rescan; ++ } ++ ++ if (column + increment > width && column != 0) ++ { ++ fwrite (line_out, sizeof(char), offset_out, stdout); ++ START_NEW_LINE; ++ goto rescan; ++ } ++ ++ if (allocated_out < offset_out + mblength) ++ { ++ line_out = X2REALLOC (line_out, &allocated_out); ++ } ++ ++ memcpy (line_out + offset_out, bufpos, mblength); ++ offset_out += mblength; ++ column += increment; ++ ++ if (is_blank_seen && !convfail && wc == L'\r') ++ is_cr_after_last_blank = 1; ++ ++ if (is_bs_following_last_blank && !convfail && wc == L'\b') ++ ++bs_following_last_blank_num; ++ else ++ is_bs_following_last_blank = 0; ++ ++ if (break_spaces && !convfail && iswblank (wc)) ++ { ++ last_blank_pos = offset_out; ++ last_blank_column = column; ++ is_blank_seen = 1; ++ last_blank_increment = increment; ++ is_bs_following_last_blank = 1; ++ bs_following_last_blank_num = 0; ++ is_cr_after_last_blank = 0; ++ } ++ } ++ ++ *saved_errno = errno; + + if (offset_out) + fwrite (line_out, sizeof (char), (size_t) offset_out, stdout); + ++} ++#endif ++ ++/* Fold file FILENAME, or standard input if FILENAME is "-", ++ to stdout, with maximum line length WIDTH. ++ Return 0 if successful, 1 if an error occurs. */ ++ ++static bool ++fold_file (char const *filename, size_t width) ++{ ++ FILE *istream; ++ int saved_errno; ++ ++ if (STREQ (filename, "-")) ++ { ++ istream = stdin; ++ have_read_stdin = 1; ++ } ++ else ++ istream = fopen (filename, "r"); ++ ++ if (istream == NULL) ++ { ++ error (0, errno, "%s", filename); ++ return 1; ++ } ++ ++ /* Define how ISTREAM is being folded. */ ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ fold_multibyte_text (istream, width, &saved_errno); ++ else ++#endif ++ fold_text (istream, width, &saved_errno); ++ + if (ferror (istream)) + { + error (0, saved_errno, "%s", quotef (filename)); +@@ -252,7 +498,8 @@ main (int argc, char **argv) + + atexit (close_stdout); + +- break_spaces = count_bytes = have_read_stdin = false; ++ operating_mode = column_mode; ++ break_spaces = have_read_stdin = false; + + while ((optc = getopt_long (argc, argv, shortopts, longopts, NULL)) != -1) + { +@@ -261,7 +508,15 @@ main (int argc, char **argv) + switch (optc) + { + case 'b': /* Count bytes rather than columns. */ +- count_bytes = true; ++ if (operating_mode != column_mode) ++ FATAL_ERROR (_("only one way of folding may be specified")); ++ operating_mode = byte_mode; ++ break; ++ ++ case 'c': ++ if (operating_mode != column_mode) ++ FATAL_ERROR (_("only one way of folding may be specified")); ++ operating_mode = character_mode; + break; + + case 's': /* Break at word boundaries. */ +diff -Naurp coreutils-8.27-orig/src/join.c coreutils-8.27/src/join.c +--- coreutils-8.27-orig/src/join.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/join.c 2017-03-11 23:47:13.091286290 -0600 +@@ -22,19 +22,33 @@ + #include + #include + ++/* Get mbstate_t, mbrtowc(), mbrtowc(), wcwidth(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ ++/* Get iswblank(), towupper. */ ++#if HAVE_WCTYPE_H ++# include ++#endif ++ + #include "system.h" + #include "die.h" + #include "error.h" + #include "fadvise.h" + #include "hard-locale.h" + #include "linebuffer.h" +-#include "memcasecmp.h" + #include "quote.h" + #include "stdio--.h" + #include "xmemcoll.h" + #include "xstrtol.h" + #include "argmatch.h" + ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ + /* The official name of this program (e.g., no 'g' prefix). */ + #define PROGRAM_NAME "join" + +@@ -136,10 +150,12 @@ static struct outlist outlist_head; + /* Last element in 'outlist', where a new element can be added. */ + static struct outlist *outlist_end = &outlist_head; + +-/* Tab character separating fields. If negative, fields are separated +- by any nonempty string of blanks, otherwise by exactly one +- tab character whose value (when cast to unsigned char) equals TAB. */ +-static int tab = -1; ++/* Tab character separating fields. If NULL, fields are separated ++ by any nonempty string of blanks. */ ++static char *tab = NULL; ++ ++/* The number of bytes used for tab. */ ++static size_t tablen = 0; + + /* If nonzero, check that the input is correctly ordered. */ + static enum +@@ -276,13 +292,14 @@ xfields (struct line *line) + if (ptr == lim) + return; + +- if (0 <= tab && tab != '\n') ++ if (tab != NULL) + { ++ unsigned char t = tab[0]; + char *sep; +- for (; (sep = memchr (ptr, tab, lim - ptr)) != NULL; ptr = sep + 1) ++ for (; (sep = memchr (ptr, t, lim - ptr)) != NULL; ptr = sep + 1) + extract_field (line, ptr, sep - ptr); + } +- else if (tab < 0) ++ else + { + /* Skip leading blanks before the first field. */ + while (field_sep (*ptr)) +@@ -306,6 +323,147 @@ xfields (struct line *line) + extract_field (line, ptr, lim - ptr); + } + ++#if HAVE_MBRTOWC ++static void ++xfields_multibyte (struct line *line) ++{ ++ char *ptr = line->buf.buffer; ++ char const *lim = ptr + line->buf.length - 1; ++ wchar_t wc = 0; ++ size_t mblength = 1; ++ mbstate_t state, state_bak; ++ ++ memset (&state, 0, sizeof (mbstate_t)); ++ ++ if (ptr >= lim) ++ return; ++ ++ if (tab != NULL) ++ { ++ char *sep = ptr; ++ for (; ptr < lim; ptr = sep + mblength) ++ { ++ sep = ptr; ++ while (sep < lim) ++ { ++ state_bak = state; ++ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state); ++ ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ if (mblength == tablen && !memcmp (sep, tab, mblength)) ++ break; ++ else ++ { ++ sep += mblength; ++ continue; ++ } ++ } ++ ++ if (sep >= lim) ++ break; ++ ++ extract_field (line, ptr, sep - ptr); ++ } ++ } ++ else ++ { ++ /* Skip leading blanks before the first field. */ ++ while(ptr < lim) ++ { ++ state_bak = state; ++ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state); ++ ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ break; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ if (!iswblank(wc) && wc != '\n') ++ break; ++ ptr += mblength; ++ } ++ ++ do ++ { ++ char *sep; ++ state_bak = state; ++ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state); ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ break; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ sep = ptr + mblength; ++ while (sep < lim) ++ { ++ state_bak = state; ++ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state); ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ break; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ if (iswblank (wc) || wc == '\n') ++ break; ++ ++ sep += mblength; ++ } ++ ++ extract_field (line, ptr, sep - ptr); ++ if (sep >= lim) ++ return; ++ ++ state_bak = state; ++ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state); ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ break; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ ptr = sep + mblength; ++ while (ptr < lim) ++ { ++ state_bak = state; ++ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state); ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ break; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ if (!iswblank (wc) && wc != '\n') ++ break; ++ ++ ptr += mblength; ++ } ++ } ++ while (ptr < lim); ++ } ++ ++ extract_field (line, ptr, lim - ptr); ++} ++#endif ++ + static void + freeline (struct line *line) + { +@@ -327,56 +485,133 @@ keycmp (struct line const *line1, struct + size_t jf_1, size_t jf_2) + { + /* Start of field to compare in each file. */ +- char *beg1; +- char *beg2; +- +- size_t len1; +- size_t len2; /* Length of fields to compare. */ ++ char *beg[2]; ++ char *copy[2]; ++ size_t len[2]; /* Length of fields to compare. */ + int diff; ++ int i, j; ++ int mallocd = 0; + + if (jf_1 < line1->nfields) + { +- beg1 = line1->fields[jf_1].beg; +- len1 = line1->fields[jf_1].len; ++ beg[0] = line1->fields[jf_1].beg; ++ len[0] = line1->fields[jf_1].len; + } + else + { +- beg1 = NULL; +- len1 = 0; ++ beg[0] = NULL; ++ len[0] = 0; + } + + if (jf_2 < line2->nfields) + { +- beg2 = line2->fields[jf_2].beg; +- len2 = line2->fields[jf_2].len; ++ beg[1] = line2->fields[jf_2].beg; ++ len[1] = line2->fields[jf_2].len; + } + else + { +- beg2 = NULL; +- len2 = 0; ++ beg[1] = NULL; ++ len[1] = 0; + } + +- if (len1 == 0) +- return len2 == 0 ? 0 : -1; +- if (len2 == 0) ++ if (len[0] == 0) ++ return len[1] == 0 ? 0 : -1; ++ if (len[1] == 0) + return 1; + + if (ignore_case) + { +- /* FIXME: ignore_case does not work with NLS (in particular, +- with multibyte chars). */ +- diff = memcasecmp (beg1, beg2, MIN (len1, len2)); ++#ifdef HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ size_t mblength; ++ wchar_t wc, uwc; ++ mbstate_t state, state_bak; ++ ++ memset (&state, '\0', sizeof (mbstate_t)); ++ ++ for (i = 0; i < 2; i++) ++ { ++ mallocd = 1; ++ copy[i] = xmalloc (len[i] + 1); ++ memset (copy[i], '\0',len[i] + 1); ++ ++ for (j = 0; j < MIN (len[0], len[1]);) ++ { ++ state_bak = state; ++ mblength = mbrtowc (&wc, beg[i] + j, len[i] - j, &state); ++ ++ switch (mblength) ++ { ++ case (size_t) -1: ++ case (size_t) -2: ++ state = state_bak; ++ /* Fall through */ ++ case 0: ++ mblength = 1; ++ break; ++ ++ default: ++ uwc = towupper (wc); ++ ++ if (uwc != wc) ++ { ++ mbstate_t state_wc; ++ size_t mblen; ++ ++ memset (&state_wc, '\0', sizeof (mbstate_t)); ++ mblen = wcrtomb (copy[i] + j, uwc, &state_wc); ++ assert (mblen != (size_t)-1); ++ } ++ else ++ memcpy (copy[i] + j, beg[i] + j, mblength); ++ } ++ j += mblength; ++ } ++ copy[i][j] = '\0'; ++ } ++ } ++ else ++#endif ++ { ++ for (i = 0; i < 2; i++) ++ { ++ mallocd = 1; ++ copy[i] = xmalloc (len[i] + 1); ++ ++ for (j = 0; j < MIN (len[0], len[1]); j++) ++ copy[i][j] = toupper (beg[i][j]); ++ ++ copy[i][j] = '\0'; ++ } ++ } + } + else + { +- if (hard_LC_COLLATE) +- return xmemcoll (beg1, len1, beg2, len2); +- diff = memcmp (beg1, beg2, MIN (len1, len2)); ++ copy[0] = beg[0]; ++ copy[1] = beg[1]; + } + ++ if (hard_LC_COLLATE) ++ { ++ diff = xmemcoll ((char *) copy[0], len[0], (char *) copy[1], len[1]); ++ ++ if (mallocd) ++ for (i = 0; i < 2; i++) ++ free (copy[i]); ++ ++ return diff; ++ } ++ diff = memcmp (copy[0], copy[1], MIN (len[0], len[1])); ++ ++ if (mallocd) ++ for (i = 0; i < 2; i++) ++ free (copy[i]); ++ ++ + if (diff) + return diff; +- return len1 < len2 ? -1 : len1 != len2; ++ return len[0] - len[1]; + } + + /* Check that successive input lines PREV and CURRENT from input file +@@ -468,6 +703,11 @@ get_line (FILE *fp, struct line **linep, + } + ++line_no[which - 1]; + ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ xfields_multibyte (line); ++ else ++#endif + xfields (line); + + if (prevline[which - 1]) +@@ -567,21 +807,28 @@ prfield (size_t n, struct line const *li + + /* Output all the fields in line, other than the join field. */ + ++#define PUT_TAB_CHAR \ ++ do \ ++ { \ ++ (tab != NULL) ? \ ++ fwrite(tab, sizeof(char), tablen, stdout) : putchar (' '); \ ++ } \ ++ while (0) ++ + static void + prfields (struct line const *line, size_t join_field, size_t autocount) + { + size_t i; + size_t nfields = autoformat ? autocount : line->nfields; +- char output_separator = tab < 0 ? ' ' : tab; + + for (i = 0; i < join_field && i < nfields; ++i) + { +- putchar (output_separator); ++ PUT_TAB_CHAR; + prfield (i, line); + } + for (i = join_field + 1; i < nfields; ++i) + { +- putchar (output_separator); ++ PUT_TAB_CHAR; + prfield (i, line); + } + } +@@ -592,7 +839,6 @@ static void + prjoin (struct line const *line1, struct line const *line2) + { + const struct outlist *outlist; +- char output_separator = tab < 0 ? ' ' : tab; + size_t field; + struct line const *line; + +@@ -626,7 +872,7 @@ prjoin (struct line const *line1, struct + o = o->next; + if (o == NULL) + break; +- putchar (output_separator); ++ PUT_TAB_CHAR; + } + putchar (eolchar); + } +@@ -1104,20 +1350,43 @@ main (int argc, char **argv) + + case 't': + { +- unsigned char newtab = optarg[0]; ++ char *newtab = NULL; ++ size_t newtablen; ++ newtab = xstrdup (optarg); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ mbstate_t state; ++ ++ memset (&state, 0, sizeof (mbstate_t)); ++ newtablen = mbrtowc (NULL, newtab, ++ strnlen (newtab, MB_LEN_MAX), ++ &state); ++ if (newtablen == (size_t) 0 ++ || newtablen == (size_t) -1 ++ || newtablen == (size_t) -2) ++ newtablen = 1; ++ } ++ else ++#endif ++ newtablen = 1; + if (! newtab) +- newtab = '\n'; /* '' => process the whole line. */ ++ newtab = (char*)"\n"; /* '' => process the whole line. */ + else if (optarg[1]) + { +- if (STREQ (optarg, "\\0")) +- newtab = '\0'; +- else +- die (EXIT_FAILURE, 0, _("multi-character tab %s"), +- quote (optarg)); ++ if (newtablen == 1 && newtab[1]) ++ { ++ if (STREQ (newtab, "\\0")) ++ newtab[0] = '\0'; ++ } ++ } ++ if (tab != NULL && strcmp (tab, newtab)) ++ { ++ free (newtab); ++ die (EXIT_FAILURE, 0, _("incompatible tabs")); + } +- if (0 <= tab && tab != newtab) +- die (EXIT_FAILURE, 0, _("incompatible tabs")); + tab = newtab; ++ tablen = newtablen; + } + break; + +diff -Naurp coreutils-8.27-orig/src/pr.c coreutils-8.27/src/pr.c +--- coreutils-8.27-orig/src/pr.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/pr.c 2017-03-11 23:47:13.094286139 -0600 +@@ -311,6 +311,24 @@ + + #include + #include ++ ++/* Get MB_LEN_MAX. */ ++#include ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX == 1 ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Get MB_CUR_MAX. */ ++#include ++ ++/* Solaris 2.5 has a bug: must be included before . */ ++/* Get mbstate_t, mbrtowc(), wcwidth(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ + #include "system.h" + #include "die.h" + #include "error.h" +@@ -324,6 +342,18 @@ + #include "xstrtol.h" + #include "xdectoint.h" + ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ ++#ifndef HAVE_DECL_WCWIDTH ++"this configure-time declaration test was not run" ++#endif ++#if !HAVE_DECL_WCWIDTH ++extern int wcwidth (); ++#endif ++ + /* The official name of this program (e.g., no 'g' prefix). */ + #define PROGRAM_NAME "pr" + +@@ -416,7 +446,20 @@ struct COLUMN + + typedef struct COLUMN COLUMN; + +-static int char_to_clump (char c); ++/* Funtion pointers to switch functions for single byte locale or for ++ multibyte locale. If multibyte functions do not exist in your sysytem, ++ these pointers always point the function for single byte locale. */ ++static void (*print_char) (char c); ++static int (*char_to_clump) (char c); ++ ++/* Functions for single byte locale. */ ++static void print_char_single (char c); ++static int char_to_clump_single (char c); ++ ++/* Functions for multibyte locale. */ ++static void print_char_multi (char c); ++static int char_to_clump_multi (char c); ++ + static bool read_line (COLUMN *p); + static bool print_page (void); + static bool print_stored (COLUMN *p); +@@ -428,6 +471,7 @@ static void add_line_number (COLUMN *p); + static void getoptnum (const char *n_str, int min, int *num, + const char *errfmt); + static void getoptarg (char *arg, char switch_char, char *character, ++ int *character_length, int *character_width, + int *number); + static void print_files (int number_of_files, char **av); + static void init_parameters (int number_of_files); +@@ -441,7 +485,6 @@ static void store_char (char c); + static void pad_down (unsigned int lines); + static void read_rest_of_line (COLUMN *p); + static void skip_read (COLUMN *p, int column_number); +-static void print_char (char c); + static void cleanup (void); + static void print_sep_string (void); + static void separator_string (const char *optarg_S); +@@ -453,7 +496,7 @@ static COLUMN *column_vector; + we store the leftmost columns contiguously in buff. + To print a line from buff, get the index of the first character + from line_vector[i], and print up to line_vector[i + 1]. */ +-static char *buff; ++static unsigned char *buff; + + /* Index of the position in buff where the next character + will be stored. */ +@@ -557,7 +600,7 @@ static int chars_per_column; + static bool untabify_input = false; + + /* (-e) The input tab character. */ +-static char input_tab_char = '\t'; ++static char input_tab_char[MB_LEN_MAX] = "\t"; + + /* (-e) Tabstops are at chars_per_tab, 2*chars_per_tab, 3*chars_per_tab, ... + where the leftmost column is 1. */ +@@ -567,7 +610,10 @@ static int chars_per_input_tab = 8; + static bool tabify_output = false; + + /* (-i) The output tab character. */ +-static char output_tab_char = '\t'; ++static char output_tab_char[MB_LEN_MAX] = "\t"; ++ ++/* (-i) The byte length of output tab character. */ ++static int output_tab_char_length = 1; + + /* (-i) The width of the output tab. */ + static int chars_per_output_tab = 8; +@@ -637,7 +683,13 @@ static int line_number; + static bool numbered_lines = false; + + /* (-n) Character which follows each line number. */ +-static char number_separator = '\t'; ++static char number_separator[MB_LEN_MAX] = "\t"; ++ ++/* (-n) The byte length of the character which follows each line number. */ ++static int number_separator_length = 1; ++ ++/* (-n) The character width of the character which follows each line number. */ ++static int number_separator_width = 0; + + /* (-n) line counting starts with 1st line of input file (not with 1st + line of 1st page printed). */ +@@ -690,6 +742,7 @@ static bool use_col_separator = false; + -a|COLUMN|-m is a 'space' and with the -J option a 'tab'. */ + static char const *col_sep_string = ""; + static int col_sep_length = 0; ++static int col_sep_width = 0; + static char *column_separator = (char *) " "; + static char *line_separator = (char *) "\t"; + +@@ -851,6 +904,13 @@ separator_string (const char *optarg_S) + integer_overflow (); + col_sep_length = len; + col_sep_string = optarg_S; ++ ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ col_sep_width = mbswidth (col_sep_string, 0); ++ else ++#endif ++ col_sep_width = col_sep_length; + } + + int +@@ -875,6 +935,21 @@ main (int argc, char **argv) + + atexit (close_stdout); + ++/* Define which functions are used, the ones for single byte locale or the ones ++ for multibyte locale. */ ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ print_char = print_char_multi; ++ char_to_clump = char_to_clump_multi; ++ } ++ else ++#endif ++ { ++ print_char = print_char_single; ++ char_to_clump = char_to_clump_single; ++ } ++ + n_files = 0; + file_names = (argc > 1 + ? xnmalloc (argc - 1, sizeof (char *)) +@@ -951,8 +1026,12 @@ main (int argc, char **argv) + break; + case 'e': + if (optarg) +- getoptarg (optarg, 'e', &input_tab_char, +- &chars_per_input_tab); ++ { ++ int dummy_length, dummy_width; ++ ++ getoptarg (optarg, 'e', input_tab_char, &dummy_length, ++ &dummy_width, &chars_per_input_tab); ++ } + /* Could check tab width > 0. */ + untabify_input = true; + break; +@@ -965,8 +1044,12 @@ main (int argc, char **argv) + break; + case 'i': + if (optarg) +- getoptarg (optarg, 'i', &output_tab_char, +- &chars_per_output_tab); ++ { ++ int dummy_width; ++ ++ getoptarg (optarg, 'i', output_tab_char, &output_tab_char_length, ++ &dummy_width, &chars_per_output_tab); ++ } + /* Could check tab width > 0. */ + tabify_output = true; + break; +@@ -984,8 +1067,8 @@ main (int argc, char **argv) + case 'n': + numbered_lines = true; + if (optarg) +- getoptarg (optarg, 'n', &number_separator, +- &chars_per_number); ++ getoptarg (optarg, 'n', number_separator, &number_separator_length, ++ &number_separator_width, &chars_per_number); + break; + case 'N': + skip_count = false; +@@ -1010,6 +1093,7 @@ main (int argc, char **argv) + /* Reset an additional input of -s, -S dominates -s */ + col_sep_string = ""; + col_sep_length = 0; ++ col_sep_width = 0; + use_col_separator = true; + if (optarg) + separator_string (optarg); +@@ -1166,10 +1250,45 @@ getoptnum (const char *n_str, int min, i + a number. */ + + static void +-getoptarg (char *arg, char switch_char, char *character, int *number) ++getoptarg (char *arg, char switch_char, char *character, int *character_length, ++ int *character_width, int *number) + { + if (!ISDIGIT (*arg)) +- *character = *arg++; ++ { ++#ifdef HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) /* for multibyte locale. */ ++ { ++ wchar_t wc; ++ size_t mblength; ++ int width; ++ mbstate_t state = {'\0'}; ++ ++ mblength = mbrtowc (&wc, arg, strnlen(arg, MB_LEN_MAX), &state); ++ ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ *character_length = 1; ++ *character_width = 1; ++ } ++ else ++ { ++ *character_length = (mblength < 1) ? 1 : mblength; ++ width = wcwidth (wc); ++ *character_width = (width < 0) ? 0 : width; ++ } ++ ++ strncpy (character, arg, *character_length); ++ arg += *character_length; ++ } ++ else /* for single byte locale. */ ++#endif ++ { ++ *character = *arg++; ++ *character_length = 1; ++ *character_width = 1; ++ } ++ } ++ + if (*arg) + { + long int tmp_long; +@@ -1191,6 +1310,11 @@ static void + init_parameters (int number_of_files) + { + int chars_used_by_number = 0; ++ int mb_len = 1; ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ mb_len = MB_LEN_MAX; ++#endif + + lines_per_body = lines_per_page - lines_per_header - lines_per_footer; + if (lines_per_body <= 0) +@@ -1228,7 +1352,7 @@ init_parameters (int number_of_files) + else + col_sep_string = column_separator; + +- col_sep_length = 1; ++ col_sep_length = col_sep_width = 1; + use_col_separator = true; + } + /* It's rather pointless to define a TAB separator with column +@@ -1258,11 +1382,11 @@ init_parameters (int number_of_files) + + TAB_WIDTH (chars_per_input_tab, chars_per_number); */ + + /* Estimate chars_per_text without any margin and keep it constant. */ +- if (number_separator == '\t') ++ if (number_separator[0] == '\t') + number_width = (chars_per_number + + TAB_WIDTH (chars_per_default_tab, chars_per_number)); + else +- number_width = chars_per_number + 1; ++ number_width = chars_per_number + number_separator_width; + + /* The number is part of the column width unless we are + printing files in parallel. */ +@@ -1271,7 +1395,7 @@ init_parameters (int number_of_files) + } + + int sep_chars, useful_chars; +- if (INT_MULTIPLY_WRAPV (columns - 1, col_sep_length, &sep_chars)) ++ if (INT_MULTIPLY_WRAPV (columns - 1, col_sep_width, &sep_chars)) + sep_chars = INT_MAX; + if (INT_SUBTRACT_WRAPV (chars_per_line - chars_used_by_number, sep_chars, + &useful_chars)) +@@ -1294,7 +1418,7 @@ init_parameters (int number_of_files) + We've to use 8 as the lower limit, if we use chars_per_default_tab = 8 + to expand a tab which is not an input_tab-char. */ + free (clump_buff); +- clump_buff = xmalloc (MAX (8, chars_per_input_tab)); ++ clump_buff = xmalloc (mb_len * MAX (8, chars_per_input_tab)); + } + + /* Open the necessary files, +@@ -1402,7 +1526,7 @@ init_funcs (void) + + /* Enlarge p->start_position of first column to use the same form of + padding_not_printed with all columns. */ +- h = h + col_sep_length; ++ h = h + col_sep_width; + + /* This loop takes care of all but the rightmost column. */ + +@@ -1436,7 +1560,7 @@ init_funcs (void) + } + else + { +- h = h_next + col_sep_length; ++ h = h_next + col_sep_width; + h_next = h + chars_per_column; + } + } +@@ -1727,9 +1851,9 @@ static void + align_column (COLUMN *p) + { + padding_not_printed = p->start_position; +- if (col_sep_length < padding_not_printed) ++ if (col_sep_width < padding_not_printed) + { +- pad_across_to (padding_not_printed - col_sep_length); ++ pad_across_to (padding_not_printed - col_sep_width); + padding_not_printed = ANYWHERE; + } + +@@ -2004,13 +2128,13 @@ store_char (char c) + /* May be too generous. */ + buff = X2REALLOC (buff, &buff_allocated); + } +- buff[buff_current++] = c; ++ buff[buff_current++] = (unsigned char) c; + } + + static void + add_line_number (COLUMN *p) + { +- int i; ++ int i, j; + char *s; + int num_width; + +@@ -2027,22 +2151,24 @@ add_line_number (COLUMN *p) + /* Tabification is assumed for multiple columns, also for n-separators, + but 'default n-separator = TAB' hasn't been given priority over + equal column_width also specified by POSIX. */ +- if (number_separator == '\t') ++ if (number_separator[0] == '\t') + { + i = number_width - chars_per_number; + while (i-- > 0) + (p->char_func) (' '); + } + else +- (p->char_func) (number_separator); ++ for (j = 0; j < number_separator_length; j++) ++ (p->char_func) (number_separator[j]); + } + else + /* To comply with POSIX, we avoid any expansion of default TAB + separator with a single column output. No column_width requirement + has to be considered. */ + { +- (p->char_func) (number_separator); +- if (number_separator == '\t') ++ for (j = 0; j < number_separator_length; j++) ++ (p->char_func) (number_separator[j]); ++ if (number_separator[0] == '\t') + output_position = POS_AFTER_TAB (chars_per_output_tab, + output_position); + } +@@ -2203,7 +2329,7 @@ print_white_space (void) + while (goal - h_old > 1 + && (h_new = POS_AFTER_TAB (chars_per_output_tab, h_old)) <= goal) + { +- putchar (output_tab_char); ++ fwrite (output_tab_char, sizeof(char), output_tab_char_length, stdout); + h_old = h_new; + } + while (++h_old <= goal) +@@ -2223,6 +2349,7 @@ print_sep_string (void) + { + char const *s = col_sep_string; + int l = col_sep_length; ++ int not_space_flag; + + if (separators_not_printed <= 0) + { +@@ -2234,6 +2361,7 @@ print_sep_string (void) + { + for (; separators_not_printed > 0; --separators_not_printed) + { ++ not_space_flag = 0; + while (l-- > 0) + { + /* 3 types of sep_strings: spaces only, spaces and chars, +@@ -2247,12 +2375,15 @@ print_sep_string (void) + } + else + { ++ not_space_flag = 1; + if (spaces_not_printed > 0) + print_white_space (); + putchar (*s++); +- ++output_position; + } + } ++ if (not_space_flag) ++ output_position += col_sep_width; ++ + /* sep_string ends with some spaces */ + if (spaces_not_printed > 0) + print_white_space (); +@@ -2280,7 +2411,7 @@ print_clump (COLUMN *p, int n, char *clu + required number of tabs and spaces. */ + + static void +-print_char (char c) ++print_char_single (char c) + { + if (tabify_output) + { +@@ -2304,6 +2435,74 @@ print_char (char c) + putchar (c); + } + ++#ifdef HAVE_MBRTOWC ++static void ++print_char_multi (char c) ++{ ++ static size_t mbc_pos = 0; ++ static char mbc[MB_LEN_MAX] = {'\0'}; ++ static mbstate_t state = {'\0'}; ++ mbstate_t state_bak; ++ wchar_t wc; ++ size_t mblength; ++ int width; ++ ++ if (tabify_output) ++ { ++ state_bak = state; ++ mbc[mbc_pos++] = c; ++ mblength = mbrtowc (&wc, mbc, mbc_pos, &state); ++ ++ while (mbc_pos > 0) ++ { ++ switch (mblength) ++ { ++ case (size_t)-2: ++ state = state_bak; ++ return; ++ ++ case (size_t)-1: ++ state = state_bak; ++ ++output_position; ++ putchar (mbc[0]); ++ memmove (mbc, mbc + 1, MB_CUR_MAX - 1); ++ --mbc_pos; ++ break; ++ ++ case 0: ++ mblength = 1; ++ ++ default: ++ if (wc == L' ') ++ { ++ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength); ++ --mbc_pos; ++ ++spaces_not_printed; ++ return; ++ } ++ else if (spaces_not_printed > 0) ++ print_white_space (); ++ ++ /* Nonprintables are assumed to have width 0, except L'\b'. */ ++ if ((width = wcwidth (wc)) < 1) ++ { ++ if (wc == L'\b') ++ --output_position; ++ } ++ else ++ output_position += width; ++ ++ fwrite (mbc, sizeof(char), mblength, stdout); ++ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength); ++ mbc_pos -= mblength; ++ } ++ } ++ return; ++ } ++ putchar (c); ++} ++#endif ++ + /* Skip to page PAGE before printing. + PAGE may be larger than total number of pages. */ + +@@ -2483,9 +2682,9 @@ read_line (COLUMN *p) + align_empty_cols = false; + } + +- if (col_sep_length < padding_not_printed) ++ if (col_sep_width < padding_not_printed) + { +- pad_across_to (padding_not_printed - col_sep_length); ++ pad_across_to (padding_not_printed - col_sep_width); + padding_not_printed = ANYWHERE; + } + +@@ -2555,7 +2754,7 @@ print_stored (COLUMN *p) + int i; + + int line = p->current_line++; +- char *first = &buff[line_vector[line]]; ++ unsigned char *first = &buff[line_vector[line]]; + /* FIXME + UMR: Uninitialized memory read: + * This is occurring while in: +@@ -2567,7 +2766,7 @@ print_stored (COLUMN *p) + xmalloc [xmalloc.c:94] + init_store_cols [pr.c:1648] + */ +- char *last = &buff[line_vector[line + 1]]; ++ unsigned char *last = &buff[line_vector[line + 1]]; + + pad_vertically = true; + +@@ -2586,9 +2785,9 @@ print_stored (COLUMN *p) + } + } + +- if (col_sep_length < padding_not_printed) ++ if (col_sep_width < padding_not_printed) + { +- pad_across_to (padding_not_printed - col_sep_length); ++ pad_across_to (padding_not_printed - col_sep_width); + padding_not_printed = ANYWHERE; + } + +@@ -2601,8 +2800,8 @@ print_stored (COLUMN *p) + if (spaces_not_printed == 0) + { + output_position = p->start_position + end_vector[line]; +- if (p->start_position - col_sep_length == chars_per_margin) +- output_position -= col_sep_length; ++ if (p->start_position - col_sep_width == chars_per_margin) ++ output_position -= col_sep_width; + } + + return true; +@@ -2621,7 +2820,7 @@ print_stored (COLUMN *p) + number of characters is 1.) */ + + static int +-char_to_clump (char c) ++char_to_clump_single (char c) + { + unsigned char uc = c; + char *s = clump_buff; +@@ -2631,10 +2830,10 @@ char_to_clump (char c) + int chars; + int chars_per_c = 8; + +- if (c == input_tab_char) ++ if (c == input_tab_char[0]) + chars_per_c = chars_per_input_tab; + +- if (c == input_tab_char || c == '\t') ++ if (c == input_tab_char[0] || c == '\t') + { + width = TAB_WIDTH (chars_per_c, input_position); + +@@ -2715,6 +2914,164 @@ char_to_clump (char c) + return chars; + } + ++#ifdef HAVE_MBRTOWC ++static int ++char_to_clump_multi (char c) ++{ ++ static size_t mbc_pos = 0; ++ static char mbc[MB_LEN_MAX] = {'\0'}; ++ static mbstate_t state = {'\0'}; ++ mbstate_t state_bak; ++ wchar_t wc; ++ size_t mblength; ++ int wc_width; ++ register char *s = clump_buff; ++ register int i, j; ++ char esc_buff[4]; ++ int width; ++ int chars; ++ int chars_per_c = 8; ++ ++ state_bak = state; ++ mbc[mbc_pos++] = c; ++ mblength = mbrtowc (&wc, mbc, mbc_pos, &state); ++ ++ width = 0; ++ chars = 0; ++ while (mbc_pos > 0) ++ { ++ switch (mblength) ++ { ++ case (size_t)-2: ++ state = state_bak; ++ return 0; ++ ++ case (size_t)-1: ++ state = state_bak; ++ mblength = 1; ++ ++ if (use_esc_sequence || use_cntrl_prefix) ++ { ++ width = +4; ++ chars = +4; ++ *s++ = '\\'; ++ sprintf (esc_buff, "%03o", (unsigned char) mbc[0]); ++ for (i = 0; i <= 2; ++i) ++ *s++ = (int) esc_buff[i]; ++ } ++ else ++ { ++ width += 1; ++ chars += 1; ++ *s++ = mbc[0]; ++ } ++ break; ++ ++ case 0: ++ mblength = 1; ++ /* Fall through */ ++ ++ default: ++ if (memcmp (mbc, input_tab_char, mblength) == 0) ++ chars_per_c = chars_per_input_tab; ++ ++ if (memcmp (mbc, input_tab_char, mblength) == 0 || c == '\t') ++ { ++ int width_inc; ++ ++ width_inc = TAB_WIDTH (chars_per_c, input_position); ++ width += width_inc; ++ ++ if (untabify_input) ++ { ++ for (i = width_inc; i; --i) ++ *s++ = ' '; ++ chars += width_inc; ++ } ++ else ++ { ++ for (i = 0; i < mblength; i++) ++ *s++ = mbc[i]; ++ chars += mblength; ++ } ++ } ++ else if ((wc_width = wcwidth (wc)) < 1) ++ { ++ if (use_esc_sequence) ++ { ++ for (i = 0; i < mblength; i++) ++ { ++ width += 4; ++ chars += 4; ++ *s++ = '\\'; ++ sprintf (esc_buff, "%03o", (unsigned char) mbc[i]); ++ for (j = 0; j <= 2; ++j) ++ *s++ = (int) esc_buff[j]; ++ } ++ } ++ else if (use_cntrl_prefix) ++ { ++ if (wc < 0200) ++ { ++ width += 2; ++ chars += 2; ++ *s++ = '^'; ++ *s++ = wc ^ 0100; ++ } ++ else ++ { ++ for (i = 0; i < mblength; i++) ++ { ++ width += 4; ++ chars += 4; ++ *s++ = '\\'; ++ sprintf (esc_buff, "%03o", (unsigned char) mbc[i]); ++ for (j = 0; j <= 2; ++j) ++ *s++ = (int) esc_buff[j]; ++ } ++ } ++ } ++ else if (wc == L'\b') ++ { ++ width += -1; ++ chars += 1; ++ *s++ = c; ++ } ++ else ++ { ++ width += 0; ++ chars += mblength; ++ for (i = 0; i < mblength; i++) ++ *s++ = mbc[i]; ++ } ++ } ++ else ++ { ++ width += wc_width; ++ chars += mblength; ++ for (i = 0; i < mblength; i++) ++ *s++ = mbc[i]; ++ } ++ } ++ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength); ++ mbc_pos -= mblength; ++ } ++ ++ /* Too many backspaces must put us in position 0 -- never negative. */ ++ if (width < 0 && input_position == 0) ++ { ++ chars = 0; ++ input_position = 0; ++ } ++ else if (width < 0 && input_position <= -width) ++ input_position = 0; ++ else ++ input_position += width; ++ ++ return chars; ++} ++#endif ++ + /* We've just printed some files and need to clean up things before + looking for more options and printing the next batch of files. + +diff -Naurp coreutils-8.27-orig/src/sort.c coreutils-8.27/src/sort.c +--- coreutils-8.27-orig/src/sort.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/sort.c 2017-03-11 23:49:22.416505389 -0600 +@@ -29,6 +29,14 @@ + #include + #include + #include ++#if HAVE_WCHAR_H ++# include ++#endif ++/* Get isw* functions. */ ++#if HAVE_WCTYPE_H ++# include ++#endif ++ + #include "system.h" + #include "argmatch.h" + #include "die.h" +@@ -165,14 +173,39 @@ static int decimal_point; + /* Thousands separator; if -1, then there isn't one. */ + static int thousands_sep; + ++/* True if -f is specified. */ ++static bool folding; ++ + /* Nonzero if the corresponding locales are hard. */ + static bool hard_LC_COLLATE; +-#if HAVE_NL_LANGINFO ++#if HAVE_LANGINFO_CODESET + static bool hard_LC_TIME; + #endif + + #define NONZERO(x) ((x) != 0) + ++/* get a multibyte character's byte length. */ ++#define GET_BYTELEN_OF_CHAR(LIM, PTR, MBLENGTH, STATE) \ ++ do \ ++ { \ ++ wchar_t wc; \ ++ mbstate_t state_bak; \ ++ \ ++ state_bak = STATE; \ ++ mblength = mbrtowc (&wc, PTR, LIM - PTR, &STATE); \ ++ \ ++ switch (MBLENGTH) \ ++ { \ ++ case (size_t)-1: \ ++ case (size_t)-2: \ ++ STATE = state_bak; \ ++ /* Fall through. */ \ ++ case 0: \ ++ MBLENGTH = 1; \ ++ } \ ++ } \ ++ while (0) ++ + /* The kind of blanks for '-b' to skip in various options. */ + enum blanktype { bl_start, bl_end, bl_both }; + +@@ -346,13 +379,11 @@ static bool reverse; + they were read if all keys compare equal. */ + static bool stable; + +-/* If TAB has this value, blanks separate fields. */ +-enum { TAB_DEFAULT = CHAR_MAX + 1 }; +- +-/* Tab character separating fields. If TAB_DEFAULT, then fields are ++/* Tab character separating fields. If tab_length is 0, then fields are + separated by the empty string between a non-blank character and a blank + character. */ +-static int tab = TAB_DEFAULT; ++static char tab[MB_LEN_MAX + 1]; ++static size_t tab_length = 0; + + /* Flag to remove consecutive duplicate lines from the output. + Only the last of a sequence of equal lines will be output. */ +@@ -811,6 +842,46 @@ reap_all (void) + reap (-1); + } + ++/* Function pointers. */ ++static void ++(*inittables) (void); ++static char * ++(*begfield) (const struct line*, const struct keyfield *); ++static char * ++(*limfield) (const struct line*, const struct keyfield *); ++static void ++(*skipblanks) (char **ptr, char *lim); ++static int ++(*getmonth) (char const *, size_t, char **); ++static int ++(*keycompare) (const struct line *, const struct line *); ++static int ++(*numcompare) (const char *, const char *); ++ ++/* Test for white space multibyte character. ++ Set LENGTH the byte length of investigated multibyte character. */ ++#if HAVE_MBRTOWC ++static int ++ismbblank (const char *str, size_t len, size_t *length) ++{ ++ size_t mblength; ++ wchar_t wc; ++ mbstate_t state; ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ mblength = mbrtowc (&wc, str, len, &state); ++ ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ *length = 1; ++ return 0; ++ } ++ ++ *length = (mblength < 1) ? 1 : mblength; ++ return iswblank (wc) || wc == '\n'; ++} ++#endif ++ + /* Clean up any remaining temporary files. */ + + static void +@@ -1255,7 +1326,7 @@ zaptemp (char const *name) + free (node); + } + +-#if HAVE_NL_LANGINFO ++#if HAVE_LANGINFO_CODESET + + static int + struct_month_cmp (void const *m1, void const *m2) +@@ -1270,7 +1341,7 @@ struct_month_cmp (void const *m1, void c + /* Initialize the character class tables. */ + + static void +-inittables (void) ++inittables_uni (void) + { + size_t i; + +@@ -1282,7 +1353,7 @@ inittables (void) + fold_toupper[i] = toupper (i); + } + +-#if HAVE_NL_LANGINFO ++#if HAVE_LANGINFO_CODESET + /* If we're not in the "C" locale, read different names for months. */ + if (hard_LC_TIME) + { +@@ -1364,6 +1435,84 @@ specify_nmerge (int oi, char c, char con + xstrtol_fatal (e, oi, c, long_options, s); + } + ++#if HAVE_MBRTOWC ++static void ++inittables_mb (void) ++{ ++ int i, j, k, l; ++ char *name, *s, *lc_time, *lc_ctype; ++ size_t s_len, mblength; ++ char mbc[MB_LEN_MAX]; ++ wchar_t wc, pwc; ++ mbstate_t state_mb, state_wc; ++ ++ lc_time = setlocale (LC_TIME, ""); ++ if (lc_time) ++ lc_time = xstrdup (lc_time); ++ ++ lc_ctype = setlocale (LC_CTYPE, ""); ++ if (lc_ctype) ++ lc_ctype = xstrdup (lc_ctype); ++ ++ if (lc_time && lc_ctype) ++ /* temporarily set LC_CTYPE to match LC_TIME, so that we can convert ++ * the names of months to upper case */ ++ setlocale (LC_CTYPE, lc_time); ++ ++ for (i = 0; i < MONTHS_PER_YEAR; i++) ++ { ++ s = (char *) nl_langinfo (ABMON_1 + i); ++ s_len = strlen (s); ++ monthtab[i].name = name = (char *) xmalloc (s_len + 1); ++ monthtab[i].val = i + 1; ++ ++ memset (&state_mb, '\0', sizeof (mbstate_t)); ++ memset (&state_wc, '\0', sizeof (mbstate_t)); ++ ++ for (j = 0; j < s_len;) ++ { ++ if (!ismbblank (s + j, s_len - j, &mblength)) ++ break; ++ j += mblength; ++ } ++ ++ for (k = 0; j < s_len;) ++ { ++ mblength = mbrtowc (&wc, (s + j), (s_len - j), &state_mb); ++ assert (mblength != (size_t)-1 && mblength != (size_t)-2); ++ if (mblength == 0) ++ break; ++ ++ pwc = towupper (wc); ++ if (pwc == wc) ++ { ++ memcpy (mbc, s + j, mblength); ++ j += mblength; ++ } ++ else ++ { ++ j += mblength; ++ mblength = wcrtomb (mbc, pwc, &state_wc); ++ assert (mblength != (size_t)0 && mblength != (size_t)-1); ++ } ++ ++ for (l = 0; l < mblength; l++) ++ name[k++] = mbc[l]; ++ } ++ name[k] = '\0'; ++ } ++ qsort ((void *) monthtab, MONTHS_PER_YEAR, ++ sizeof (struct month), struct_month_cmp); ++ ++ if (lc_time && lc_ctype) ++ /* restore the original locales */ ++ setlocale (LC_CTYPE, lc_ctype); ++ ++ free (lc_ctype); ++ free (lc_time); ++} ++#endif ++ + /* Specify the amount of main memory to use when sorting. */ + static void + specify_sort_size (int oi, char c, char const *s) +@@ -1597,7 +1746,7 @@ buffer_linelim (struct buffer const *buf + by KEY in LINE. */ + + static char * +-begfield (struct line const *line, struct keyfield const *key) ++begfield_uni (const struct line *line, const struct keyfield *key) + { + char *ptr = line->text, *lim = ptr + line->length - 1; + size_t sword = key->sword; +@@ -1606,10 +1755,10 @@ begfield (struct line const *line, struc + /* The leading field separator itself is included in a field when -t + is absent. */ + +- if (tab != TAB_DEFAULT) ++ if (tab_length) + while (ptr < lim && sword--) + { +- while (ptr < lim && *ptr != tab) ++ while (ptr < lim && *ptr != tab[0]) + ++ptr; + if (ptr < lim) + ++ptr; +@@ -1635,11 +1784,70 @@ begfield (struct line const *line, struc + return ptr; + } + ++#if HAVE_MBRTOWC ++static char * ++begfield_mb (const struct line *line, const struct keyfield *key) ++{ ++ int i; ++ char *ptr = line->text, *lim = ptr + line->length - 1; ++ size_t sword = key->sword; ++ size_t schar = key->schar; ++ size_t mblength; ++ mbstate_t state; ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ if (tab_length) ++ while (ptr < lim && sword--) ++ { ++ while (ptr < lim && memcmp (ptr, tab, tab_length) != 0) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ if (ptr < lim) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ } ++ else ++ while (ptr < lim && sword--) ++ { ++ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ if (ptr < lim) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ } ++ ++ if (key->skipsblanks) ++ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ ++ for (i = 0; i < schar; i++) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ++ if (ptr + mblength > lim) ++ break; ++ else ++ ptr += mblength; ++ } ++ ++ return ptr; ++} ++#endif ++ + /* Return the limit of (a pointer to the first character after) the field + in LINE specified by KEY. */ + + static char * +-limfield (struct line const *line, struct keyfield const *key) ++limfield_uni (const struct line *line, const struct keyfield *key) + { + char *ptr = line->text, *lim = ptr + line->length - 1; + size_t eword = key->eword, echar = key->echar; +@@ -1654,10 +1862,10 @@ limfield (struct line const *line, struc + 'beginning' is the first character following the delimiting TAB. + Otherwise, leave PTR pointing at the first 'blank' character after + the preceding field. */ +- if (tab != TAB_DEFAULT) ++ if (tab_length) + while (ptr < lim && eword--) + { +- while (ptr < lim && *ptr != tab) ++ while (ptr < lim && *ptr != tab[0]) + ++ptr; + if (ptr < lim && (eword || echar)) + ++ptr; +@@ -1703,10 +1911,10 @@ limfield (struct line const *line, struc + */ + + /* Make LIM point to the end of (one byte past) the current field. */ +- if (tab != TAB_DEFAULT) ++ if (tab_length) + { + char *newlim; +- newlim = memchr (ptr, tab, lim - ptr); ++ newlim = memchr (ptr, tab[0], lim - ptr); + if (newlim) + lim = newlim; + } +@@ -1737,6 +1945,130 @@ limfield (struct line const *line, struc + return ptr; + } + ++#if HAVE_MBRTOWC ++static char * ++limfield_mb (const struct line *line, const struct keyfield *key) ++{ ++ char *ptr = line->text, *lim = ptr + line->length - 1; ++ size_t eword = key->eword, echar = key->echar; ++ int i; ++ size_t mblength; ++ mbstate_t state; ++ ++ if (echar == 0) ++ eword++; /* skip all of end field. */ ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ if (tab_length) ++ while (ptr < lim && eword--) ++ { ++ while (ptr < lim && memcmp (ptr, tab, tab_length) != 0) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ if (ptr < lim && (eword | echar)) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ } ++ else ++ while (ptr < lim && eword--) ++ { ++ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ if (ptr < lim) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ } ++ ++ ++# ifdef POSIX_UNSPECIFIED ++ /* Make LIM point to the end of (one byte past) the current field. */ ++ if (tab_length) ++ { ++ char *newlim, *p; ++ ++ newlim = NULL; ++ for (p = ptr; p < lim;) ++ { ++ if (memcmp (p, tab, tab_length) == 0) ++ { ++ newlim = p; ++ break; ++ } ++ ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ p += mblength; ++ } ++ } ++ else ++ { ++ char *newlim; ++ newlim = ptr; ++ ++ while (newlim < lim && ismbblank (newlim, lim - newlim, &mblength)) ++ newlim += mblength; ++ if (ptr < lim) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ while (newlim < lim && !ismbblank (newlim, lim - newlim, &mblength)) ++ newlim += mblength; ++ lim = newlim; ++ } ++# endif ++ ++ if (echar != 0) ++ { ++ /* If we're skipping leading blanks, don't start counting characters ++ * until after skipping past any leading blanks. */ ++ if (key->skipeblanks) ++ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ /* Advance PTR by ECHAR (if possible), but no further than LIM. */ ++ for (i = 0; i < echar; i++) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ++ if (ptr + mblength > lim) ++ break; ++ else ++ ptr += mblength; ++ } ++ } ++ ++ return ptr; ++} ++#endif ++ ++static void ++skipblanks_uni (char **ptr, char *lim) ++{ ++ while (*ptr < lim && blanks[to_uchar (**ptr)]) ++ ++(*ptr); ++} ++ ++#if HAVE_MBRTOWC ++static void ++skipblanks_mb (char **ptr, char *lim) ++{ ++ size_t mblength; ++ while (*ptr < lim && ismbblank (*ptr, lim - *ptr, &mblength)) ++ (*ptr) += mblength; ++} ++#endif ++ + /* Fill BUF reading from FP, moving buf->left bytes from the end + of buf->buf to the beginning first. If EOF is reached and the + file wasn't terminated by a newline, supply one. Set up BUF's line +@@ -1823,8 +2155,22 @@ fillbuf (struct buffer *buf, FILE *fp, c + else + { + if (key->skipsblanks) +- while (blanks[to_uchar (*line_start)]) +- line_start++; ++ { ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ size_t mblength; ++ while (line_start < line->keylim && ++ ismbblank (line_start, ++ line->keylim - line_start, ++ &mblength)) ++ line_start += mblength; ++ } ++ else ++#endif ++ while (blanks[to_uchar (*line_start)]) ++ line_start++; ++ } + line->keybeg = line_start; + } + } +@@ -1958,12 +2304,10 @@ find_unit_order (char const *number) + < K/k < M < G < T < P < E < Z < Y */ + + static int +-human_numcompare (char const *a, char const *b) ++human_numcompare (char *a, char *b) + { +- while (blanks[to_uchar (*a)]) +- a++; +- while (blanks[to_uchar (*b)]) +- b++; ++ skipblanks(&a, a + strlen(a)); ++ skipblanks(&b, b + strlen(b)); + + int diff = find_unit_order (a) - find_unit_order (b); + return (diff ? diff : strnumcmp (a, b, decimal_point, thousands_sep)); +@@ -1974,7 +2318,7 @@ human_numcompare (char const *a, char co + hideously fast. */ + + static int +-numcompare (char const *a, char const *b) ++numcompare_uni (const char *a, const char *b) + { + while (blanks[to_uchar (*a)]) + a++; +@@ -1984,6 +2328,25 @@ numcompare (char const *a, char const *b + return strnumcmp (a, b, decimal_point, thousands_sep); + } + ++#if HAVE_MBRTOWC ++static int ++numcompare_mb (const char *a, const char *b) ++{ ++ size_t mblength, len; ++ len = strlen (a); /* okay for UTF-8 */ ++ while (*a && ismbblank (a, len > MB_CUR_MAX ? MB_CUR_MAX : len, &mblength)) ++ { ++ a += mblength; ++ len -= mblength; ++ } ++ len = strlen (b); /* okay for UTF-8 */ ++ while (*b && ismbblank (b, len > MB_CUR_MAX ? MB_CUR_MAX : len, &mblength)) ++ b += mblength; ++ ++ return strnumcmp (a, b, decimal_point, thousands_sep); ++} ++#endif /* HAV_EMBRTOWC */ ++ + /* Work around a problem whereby the long double value returned by glibc's + strtold ("NaN", ...) contains uninitialized bits: clear all bytes of + A and B before calling strtold. FIXME: remove this function once +@@ -2034,7 +2397,7 @@ general_numcompare (char const *sa, char + Return 0 if the name in S is not recognized. */ + + static int +-getmonth (char const *month, char **ea) ++getmonth_uni (char const *month, size_t len, char **ea) + { + size_t lo = 0; + size_t hi = MONTHS_PER_YEAR; +@@ -2310,15 +2673,14 @@ debug_key (struct line const *line, stru + char saved = *lim; + *lim = '\0'; + +- while (blanks[to_uchar (*beg)]) +- beg++; ++ skipblanks (&beg, lim); + + char *tighter_lim = beg; + + if (lim < beg) + tighter_lim = lim; + else if (key->month) +- getmonth (beg, &tighter_lim); ++ getmonth (beg, lim-beg, &tighter_lim); + else if (key->general_numeric) + ignore_value (strtold (beg, &tighter_lim)); + else if (key->numeric || key->human_numeric) +@@ -2452,7 +2814,7 @@ key_warnings (struct keyfield const *gke + /* Warn about significant leading blanks. */ + bool implicit_skip = key_numeric (key) || key->month; + bool line_offset = key->eword == 0 && key->echar != 0; /* -k1.x,1.y */ +- if (!zero_width && !gkey_only && tab == TAB_DEFAULT && !line_offset ++ if (!zero_width && !gkey_only && !tab_length && !line_offset + && ((!key->skipsblanks && !implicit_skip) + || (!key->skipsblanks && key->schar) + || (!key->skipeblanks && key->echar))) +@@ -2510,11 +2872,87 @@ key_warnings (struct keyfield const *gke + error (0, 0, _("option '-r' only applies to last-resort comparison")); + } + ++#if HAVE_MBRTOWC ++static int ++getmonth_mb (const char *s, size_t len, char **ea) ++{ ++ char *month; ++ register size_t i; ++ register int lo = 0, hi = MONTHS_PER_YEAR, result; ++ char *tmp; ++ size_t wclength, mblength; ++ const char *pp; ++ const wchar_t *wpp; ++ wchar_t *month_wcs; ++ mbstate_t state; ++ ++ while (len > 0 && ismbblank (s, len, &mblength)) ++ { ++ s += mblength; ++ len -= mblength; ++ } ++ ++ if (len == 0) ++ return 0; ++ ++ if (SIZE_MAX - len < 1) ++ xalloc_die (); ++ ++ month = (char *) xnmalloc (len + 1, MB_CUR_MAX); ++ ++ pp = tmp = (char *) xnmalloc (len + 1, MB_CUR_MAX); ++ memcpy (tmp, s, len); ++ tmp[len] = '\0'; ++ wpp = month_wcs = (wchar_t *) xnmalloc (len + 1, sizeof (wchar_t)); ++ memset (&state, '\0', sizeof (mbstate_t)); ++ ++ wclength = mbsrtowcs (month_wcs, &pp, len + 1, &state); ++ if (wclength == (size_t)-1 || pp != NULL) ++ error (SORT_FAILURE, 0, _("Invalid multibyte input %s."), quote(s)); ++ ++ for (i = 0; i < wclength; i++) ++ { ++ month_wcs[i] = towupper(month_wcs[i]); ++ if (iswblank (month_wcs[i])) ++ { ++ month_wcs[i] = L'\0'; ++ break; ++ } ++ } ++ ++ mblength = wcsrtombs (month, &wpp, (len + 1) * MB_CUR_MAX, &state); ++ assert (mblength != (-1) && wpp == NULL); ++ ++ do ++ { ++ int ix = (lo + hi) / 2; ++ ++ if (strncmp (month, monthtab[ix].name, strlen (monthtab[ix].name)) < 0) ++ hi = ix; ++ else ++ lo = ix; ++ } ++ while (hi - lo > 1); ++ ++ result = (!strncmp (month, monthtab[lo].name, strlen (monthtab[lo].name)) ++ ? monthtab[lo].val : 0); ++ ++ if (ea && result) ++ *ea = (char*) s + strlen (monthtab[lo].name); ++ ++ free (month); ++ free (tmp); ++ free (month_wcs); ++ ++ return result; ++} ++#endif ++ + /* Compare two lines A and B trying every key in sequence until there + are no more keys or a difference is found. */ + + static int +-keycompare (struct line const *a, struct line const *b) ++keycompare_uni (const struct line *a, const struct line *b) + { + struct keyfield *key = keylist; + +@@ -2599,7 +3037,7 @@ keycompare (struct line const *a, struct + else if (key->human_numeric) + diff = human_numcompare (ta, tb); + else if (key->month) +- diff = getmonth (ta, NULL) - getmonth (tb, NULL); ++ diff = getmonth (ta, tlena, NULL) - getmonth (tb, tlenb, NULL); + else if (key->random) + diff = compare_random (ta, tlena, tb, tlenb); + else if (key->version) +@@ -2715,6 +3153,211 @@ keycompare (struct line const *a, struct + return key->reverse ? -diff : diff; + } + ++#if HAVE_MBRTOWC ++static int ++keycompare_mb (const struct line *a, const struct line *b) ++{ ++ struct keyfield *key = keylist; ++ ++ /* For the first iteration only, the key positions have been ++ precomputed for us. */ ++ char *texta = a->keybeg; ++ char *textb = b->keybeg; ++ char *lima = a->keylim; ++ char *limb = b->keylim; ++ ++ size_t mblength_a, mblength_b; ++ wchar_t wc_a, wc_b; ++ mbstate_t state_a, state_b; ++ ++ int diff = 0; ++ ++ memset (&state_a, '\0', sizeof(mbstate_t)); ++ memset (&state_b, '\0', sizeof(mbstate_t)); ++ /* Ignore keys with start after end. */ ++ if (a->keybeg - a->keylim > 0) ++ return 0; ++ ++ ++ /* Ignore and/or translate chars before comparing. */ ++# define IGNORE_CHARS(NEW_LEN, LEN, TEXT, COPY, WC, MBLENGTH, STATE) \ ++ do \ ++ { \ ++ wchar_t uwc; \ ++ char mbc[MB_LEN_MAX]; \ ++ mbstate_t state_wc; \ ++ \ ++ for (NEW_LEN = i = 0; i < LEN;) \ ++ { \ ++ mbstate_t state_bak; \ ++ \ ++ state_bak = STATE; \ ++ MBLENGTH = mbrtowc (&WC, TEXT + i, LEN - i, &STATE); \ ++ \ ++ if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1 \ ++ || MBLENGTH == 0) \ ++ { \ ++ if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1) \ ++ STATE = state_bak; \ ++ if (!ignore) \ ++ COPY[NEW_LEN++] = TEXT[i]; \ ++ i++; \ ++ continue; \ ++ } \ ++ \ ++ if (ignore) \ ++ { \ ++ if ((ignore == nonprinting && !iswprint (WC)) \ ++ || (ignore == nondictionary \ ++ && !iswalnum (WC) && !iswblank (WC))) \ ++ { \ ++ i += MBLENGTH; \ ++ continue; \ ++ } \ ++ } \ ++ \ ++ if (translate) \ ++ { \ ++ \ ++ uwc = towupper(WC); \ ++ if (WC == uwc) \ ++ { \ ++ memcpy (mbc, TEXT + i, MBLENGTH); \ ++ i += MBLENGTH; \ ++ } \ ++ else \ ++ { \ ++ i += MBLENGTH; \ ++ WC = uwc; \ ++ memset (&state_wc, '\0', sizeof (mbstate_t)); \ ++ \ ++ MBLENGTH = wcrtomb (mbc, WC, &state_wc); \ ++ assert (MBLENGTH != (size_t)-1 && MBLENGTH != 0); \ ++ } \ ++ \ ++ for (j = 0; j < MBLENGTH; j++) \ ++ COPY[NEW_LEN++] = mbc[j]; \ ++ } \ ++ else \ ++ for (j = 0; j < MBLENGTH; j++) \ ++ COPY[NEW_LEN++] = TEXT[i++]; \ ++ } \ ++ COPY[NEW_LEN] = '\0'; \ ++ } \ ++ while (0) ++ ++ /* Actually compare the fields. */ ++ ++ for (;;) ++ { ++ /* Find the lengths. */ ++ size_t lena = lima <= texta ? 0 : lima - texta; ++ size_t lenb = limb <= textb ? 0 : limb - textb; ++ ++ char enda IF_LINT (= 0); ++ char endb IF_LINT (= 0); ++ ++ char const *translate = key->translate; ++ bool const *ignore = key->ignore; ++ ++ if (ignore || translate) ++ { ++ if (SIZE_MAX - lenb - 2 < lena) ++ xalloc_die (); ++ char *copy_a = (char *) xnmalloc (lena + lenb + 2, MB_CUR_MAX); ++ char *copy_b = copy_a + lena * MB_CUR_MAX + 1; ++ size_t new_len_a, new_len_b; ++ size_t i, j; ++ ++ IGNORE_CHARS (new_len_a, lena, texta, copy_a, ++ wc_a, mblength_a, state_a); ++ IGNORE_CHARS (new_len_b, lenb, textb, copy_b, ++ wc_b, mblength_b, state_b); ++ texta = copy_a; textb = copy_b; ++ lena = new_len_a; lenb = new_len_b; ++ } ++ else ++ { ++ /* Use the keys in-place, temporarily null-terminated. */ ++ enda = texta[lena]; texta[lena] = '\0'; ++ endb = textb[lenb]; textb[lenb] = '\0'; ++ } ++ ++ if (key->random) ++ diff = compare_random (texta, lena, textb, lenb); ++ else if (key->numeric | key->general_numeric | key->human_numeric) ++ { ++ char savea = *lima, saveb = *limb; ++ ++ *lima = *limb = '\0'; ++ diff = (key->numeric ? numcompare (texta, textb) ++ : key->general_numeric ? general_numcompare (texta, textb) ++ : human_numcompare (texta, textb)); ++ *lima = savea, *limb = saveb; ++ } ++ else if (key->version) ++ diff = filevercmp (texta, textb); ++ else if (key->month) ++ diff = getmonth (texta, lena, NULL) - getmonth (textb, lenb, NULL); ++ else if (lena == 0) ++ diff = - NONZERO (lenb); ++ else if (lenb == 0) ++ diff = 1; ++ else if (hard_LC_COLLATE && !folding) ++ { ++ diff = xmemcoll0 (texta, lena + 1, textb, lenb + 1); ++ } ++ else ++ { ++ diff = memcmp (texta, textb, MIN (lena, lenb)); ++ if (diff == 0) ++ diff = lena < lenb ? -1 : lena != lenb; ++ } ++ ++ if (ignore || translate) ++ free (texta); ++ else ++ { ++ texta[lena] = enda; ++ textb[lenb] = endb; ++ } ++ ++ if (diff) ++ goto not_equal; ++ ++ key = key->next; ++ if (! key) ++ break; ++ ++ /* Find the beginning and limit of the next field. */ ++ if (key->eword != -1) ++ lima = limfield (a, key), limb = limfield (b, key); ++ else ++ lima = a->text + a->length - 1, limb = b->text + b->length - 1; ++ ++ if (key->sword != -1) ++ texta = begfield (a, key), textb = begfield (b, key); ++ else ++ { ++ texta = a->text, textb = b->text; ++ if (key->skipsblanks) ++ { ++ while (texta < lima && ismbblank (texta, lima - texta, &mblength_a)) ++ texta += mblength_a; ++ while (textb < limb && ismbblank (textb, limb - textb, &mblength_b)) ++ textb += mblength_b; ++ } ++ } ++ } ++ ++not_equal: ++ if (key && key->reverse) ++ return -diff; ++ else ++ return diff; ++} ++#endif ++ + /* Compare two lines A and B, returning negative, zero, or positive + depending on whether A compares less than, equal to, or greater than B. */ + +@@ -2742,7 +3385,7 @@ compare (struct line const *a, struct li + diff = - NONZERO (blen); + else if (blen == 0) + diff = 1; +- else if (hard_LC_COLLATE) ++ else if (hard_LC_COLLATE && !folding) + { + /* Note xmemcoll0 is a performance enhancement as + it will not unconditionally write '\0' after the +@@ -4139,6 +4782,7 @@ set_ordering (char const *s, struct keyf + break; + case 'f': + key->translate = fold_toupper; ++ folding = true; + break; + case 'g': + key->general_numeric = true; +@@ -4218,7 +4862,7 @@ main (int argc, char **argv) + initialize_exit_failure (SORT_FAILURE); + + hard_LC_COLLATE = hard_locale (LC_COLLATE); +-#if HAVE_NL_LANGINFO ++#if HAVE_LANGINFO_CODESET + hard_LC_TIME = hard_locale (LC_TIME); + #endif + +@@ -4239,6 +4883,29 @@ main (int argc, char **argv) + thousands_sep = -1; + } + ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ inittables = inittables_mb; ++ begfield = begfield_mb; ++ limfield = limfield_mb; ++ skipblanks = skipblanks_mb; ++ getmonth = getmonth_mb; ++ keycompare = keycompare_mb; ++ numcompare = numcompare_mb; ++ } ++ else ++#endif ++ { ++ inittables = inittables_uni; ++ begfield = begfield_uni; ++ limfield = limfield_uni; ++ skipblanks = skipblanks_uni; ++ getmonth = getmonth_uni; ++ keycompare = keycompare_uni; ++ numcompare = numcompare_uni; ++ } ++ + have_read_stdin = false; + inittables (); + +@@ -4513,13 +5180,34 @@ main (int argc, char **argv) + + case 't': + { +- char newtab = optarg[0]; +- if (! newtab) ++ char newtab[MB_LEN_MAX + 1]; ++ size_t newtab_length = 1; ++ strncpy (newtab, optarg, MB_LEN_MAX); ++ if (! newtab[0]) + die (SORT_FAILURE, 0, _("empty tab")); +- if (optarg[1]) ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ wchar_t wc; ++ mbstate_t state; ++ ++ memset (&state, '\0', sizeof (mbstate_t)); ++ newtab_length = mbrtowc (&wc, newtab, strnlen (newtab, ++ MB_LEN_MAX), ++ &state); ++ switch (newtab_length) ++ { ++ case (size_t) -1: ++ case (size_t) -2: ++ case 0: ++ newtab_length = 1; ++ } ++ } ++#endif ++ if (newtab_length == 1 && optarg[1]) + { + if (STREQ (optarg, "\\0")) +- newtab = '\0'; ++ newtab[0] = '\0'; + else + { + /* Provoke with 'sort -txx'. Complain about +@@ -4530,9 +5218,11 @@ main (int argc, char **argv) + quote (optarg)); + } + } +- if (tab != TAB_DEFAULT && tab != newtab) ++ if (tab_length && (tab_length != newtab_length ++ || memcmp (tab, newtab, tab_length) != 0)) + die (SORT_FAILURE, 0, _("incompatible tabs")); +- tab = newtab; ++ memcpy (tab, newtab, newtab_length); ++ tab_length = newtab_length; + } + break; + +@@ -4770,12 +5460,10 @@ main (int argc, char **argv) + sort (files, nfiles, outfile, nthreads); + } + +-#ifdef lint + if (files_from) + readtokens0_free (&tok); + else + free (files); +-#endif + + if (have_read_stdin && fclose (stdin) == EOF) + sort_die (_("close failed"), "-"); +diff -Naurp coreutils-8.27-orig/src/unexpand.c coreutils-8.27/src/unexpand.c +--- coreutils-8.27-orig/src/unexpand.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/unexpand.c 2017-03-11 23:49:06.758133530 -0600 +@@ -38,6 +38,9 @@ + #include + #include + #include ++ ++#include ++ + #include "system.h" + #include "die.h" + #include "xstrndup.h" +@@ -107,24 +110,47 @@ unexpand (void) + { + /* Input stream. */ + FILE *fp = next_file (NULL); ++ mb_file_t mbf; + + /* The array of pending blanks. In non-POSIX locales, blanks can + include characters other than spaces, so the blanks must be + stored, not merely counted. */ +- char *pending_blank; ++ mbf_char_t *pending_blank; ++ /* True if the starting locale is utf8. */ ++ bool using_utf_locale; ++ ++ /* True if the first file contains BOM header. */ ++ bool found_bom; ++ using_utf_locale=check_utf_locale(); + + if (!fp) + return; ++ mbf_init (mbf, fp); ++ found_bom=check_bom(fp,&mbf); + ++ if (using_utf_locale == false && found_bom == true) ++ { ++ /*try using some predefined locale */ ++ ++ if (set_utf_locale () != 0) ++ { ++ error (EXIT_FAILURE, errno, _("cannot set UTF-8 locale")); ++ } ++ } + /* The worst case is a non-blank character, then one blank, then a + tab stop, then MAX_COLUMN_WIDTH - 1 blanks, then a non-blank; so + allocate MAX_COLUMN_WIDTH bytes to store the blanks. */ +- pending_blank = xmalloc (max_column_width); ++ pending_blank = xmalloc (max_column_width * sizeof (mbf_char_t)); ++ ++ if (found_bom == true) ++ { ++ print_bom(); ++ } + + while (true) + { + /* Input character, or EOF. */ +- int c; ++ mbf_char_t c; + + /* If true, perform translations. */ + bool convert = true; +@@ -158,12 +184,44 @@ unexpand (void) + + do + { +- while ((c = getc (fp)) < 0 && (fp = next_file (fp))) +- continue; ++ while (true) { ++ mbf_getc (c, mbf); ++ if ((mb_iseof (c)) && (fp = next_file (fp))) ++ { ++ mbf_init (mbf, fp); ++ if (fp!=NULL) ++ { ++ if (check_bom(fp,&mbf)==true) ++ { ++ /*Not the first file - check BOM header*/ ++ if (using_utf_locale==false && found_bom==false) ++ { ++ /*BOM header in subsequent file but not in the first one. */ ++ error (EXIT_FAILURE, errno, _("combination of files with and without BOM header")); ++ } ++ } ++ else ++ { ++ if(using_utf_locale==false && found_bom==true) ++ { ++ /*First file conatined BOM header - locale was switched to UTF ++ /*all subsequent files should contain BOM. */ ++ error (EXIT_FAILURE, errno, _("combination of files with and without BOM header")); ++ } ++ } ++ } ++ continue; ++ } ++ else ++ { ++ break; ++ } ++ } ++ + + if (convert) + { +- bool blank = !! isblank (c); ++ bool blank = mb_isblank (c); + + if (blank) + { +@@ -180,16 +238,16 @@ unexpand (void) + if (next_tab_column < column) + die (EXIT_FAILURE, 0, _("input line is too long")); + +- if (c == '\t') ++ if (mb_iseq (c, '\t')) + { + column = next_tab_column; + + if (pending) +- pending_blank[0] = '\t'; ++ mb_setascii (&pending_blank[0], '\t'); + } + else + { +- column++; ++ column += mb_width (c); + + if (! (prev_blank && column == next_tab_column)) + { +@@ -197,13 +255,14 @@ unexpand (void) + will be replaced by tabs. */ + if (column == next_tab_column) + one_blank_before_tab_stop = true; +- pending_blank[pending++] = c; ++ mb_copy (&pending_blank[pending++], &c); + prev_blank = true; + continue; + } + + /* Replace the pending blanks by a tab or two. */ +- pending_blank[0] = c = '\t'; ++ mb_setascii (&c, '\t'); ++ mb_setascii (&pending_blank[0], '\t'); + } + + /* Discard pending blanks, unless it was a single +@@ -211,7 +270,7 @@ unexpand (void) + pending = one_blank_before_tab_stop; + } + } +- else if (c == '\b') ++ else if (mb_iseq (c, '\b')) + { + /* Go back one column, and force recalculation of the + next tab stop. */ +@@ -219,9 +278,9 @@ unexpand (void) + next_tab_column = column; + tab_index -= !!tab_index; + } +- else ++ else if (!mb_iseq (c, '\n')) + { +- column++; ++ column += mb_width (c); + if (!column) + die (EXIT_FAILURE, 0, _("input line is too long")); + } +@@ -229,8 +288,11 @@ unexpand (void) + if (pending) + { + if (pending > 1 && one_blank_before_tab_stop) +- pending_blank[0] = '\t'; +- if (fwrite (pending_blank, 1, pending, stdout) != pending) ++ mb_setascii (&pending_blank[0], '\t'); ++ ++ for (int n = 0; n < pending; ++n) ++ mb_putc (pending_blank[n], stdout); ++ if (ferror (stdout)) + die (EXIT_FAILURE, errno, _("write error")); + pending = 0; + one_blank_before_tab_stop = false; +@@ -240,16 +302,17 @@ unexpand (void) + convert &= convert_entire_line || blank; + } + +- if (c < 0) ++ if (mb_iseof (c)) + { + free (pending_blank); + return; + } + +- if (putchar (c) < 0) ++ mb_putc (c, stdout); ++ if (ferror (stdout)) + die (EXIT_FAILURE, errno, _("write error")); + } +- while (c != '\n'); ++ while (!mb_iseq (c, '\n')); + } + } + +diff -Naurp coreutils-8.27-orig/src/uniq.c coreutils-8.27/src/uniq.c +--- coreutils-8.27-orig/src/uniq.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/uniq.c 2017-03-11 23:47:13.098285938 -0600 +@@ -21,6 +21,17 @@ + #include + #include + ++/* Get mbstate_t, mbrtowc(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ ++/* Get isw* functions. */ ++#if HAVE_WCTYPE_H ++# include ++#endif ++#include ++ + #include "system.h" + #include "argmatch.h" + #include "linebuffer.h" +@@ -32,9 +43,21 @@ + #include "stdio--.h" + #include "xmemcoll.h" + #include "xstrtol.h" +-#include "memcasecmp.h" ++#include "xmemcoll.h" + #include "quote.h" + ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ ++ + /* The official name of this program (e.g., no 'g' prefix). */ + #define PROGRAM_NAME "uniq" + +@@ -144,6 +167,10 @@ enum + GROUP_OPTION = CHAR_MAX + 1 + }; + ++/* Function pointers. */ ++static char * ++(*find_field) (struct linebuffer *line); ++ + static struct option const longopts[] = + { + {"count", no_argument, NULL, 'c'}, +@@ -260,7 +287,7 @@ size_opt (char const *opt, char const *m + return a pointer to the beginning of the line's field to be compared. */ + + static char * _GL_ATTRIBUTE_PURE +-find_field (struct linebuffer const *line) ++find_field_uni (struct linebuffer *line) + { + size_t count; + char const *lp = line->buffer; +@@ -280,6 +307,83 @@ find_field (struct linebuffer const *lin + return line->buffer + i; + } + ++#if HAVE_MBRTOWC ++ ++# define MBCHAR_TO_WCHAR(WC, MBLENGTH, LP, POS, SIZE, STATEP, CONVFAIL) \ ++ do \ ++ { \ ++ mbstate_t state_bak; \ ++ \ ++ CONVFAIL = 0; \ ++ state_bak = *STATEP; \ ++ \ ++ MBLENGTH = mbrtowc (&WC, LP + POS, SIZE - POS, STATEP); \ ++ \ ++ switch (MBLENGTH) \ ++ { \ ++ case (size_t)-2: \ ++ case (size_t)-1: \ ++ *STATEP = state_bak; \ ++ CONVFAIL++; \ ++ /* Fall through */ \ ++ case 0: \ ++ MBLENGTH = 1; \ ++ } \ ++ } \ ++ while (0) ++ ++static char * ++find_field_multi (struct linebuffer *line) ++{ ++ size_t count; ++ char *lp = line->buffer; ++ size_t size = line->length - 1; ++ size_t pos; ++ size_t mblength; ++ wchar_t wc; ++ mbstate_t *statep; ++ int convfail = 0; ++ ++ pos = 0; ++ statep = &(line->state); ++ ++ /* skip fields. */ ++ for (count = 0; count < skip_fields && pos < size; count++) ++ { ++ while (pos < size) ++ { ++ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); ++ ++ if (convfail || !(iswblank (wc) || wc == '\n')) ++ { ++ pos += mblength; ++ break; ++ } ++ pos += mblength; ++ } ++ ++ while (pos < size) ++ { ++ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); ++ ++ if (!convfail && (iswblank (wc) || wc == '\n')) ++ break; ++ ++ pos += mblength; ++ } ++ } ++ ++ /* skip fields. */ ++ for (count = 0; count < skip_chars && pos < size; count++) ++ { ++ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); ++ pos += mblength; ++ } ++ ++ return lp + pos; ++} ++#endif ++ + /* Return false if two strings OLD and NEW match, true if not. + OLD and NEW point not to the beginnings of the lines + but rather to the beginnings of the fields to compare. +@@ -288,6 +392,8 @@ find_field (struct linebuffer const *lin + static bool + different (char *old, char *new, size_t oldlen, size_t newlen) + { ++ char *copy_old, *copy_new; ++ + if (check_chars < oldlen) + oldlen = check_chars; + if (check_chars < newlen) +@@ -295,14 +401,103 @@ different (char *old, char *new, size_t + + if (ignore_case) + { +- /* FIXME: This should invoke strcoll somehow. */ +- return oldlen != newlen || memcasecmp (old, new, oldlen); ++ size_t i; ++ ++ copy_old = xmalloc (oldlen + 1); ++ copy_new = xmalloc (oldlen + 1); ++ ++ for (i = 0; i < oldlen; i++) ++ { ++ copy_old[i] = toupper (old[i]); ++ copy_new[i] = toupper (new[i]); ++ } ++ bool rc = xmemcoll (copy_old, oldlen, copy_new, newlen); ++ free (copy_old); ++ free (copy_new); ++ return rc; + } +- else if (hard_LC_COLLATE) +- return xmemcoll (old, oldlen, new, newlen) != 0; + else +- return oldlen != newlen || memcmp (old, new, oldlen); ++ { ++ copy_old = (char *)old; ++ copy_new = (char *)new; ++ } ++ ++ return xmemcoll (copy_old, oldlen, copy_new, newlen); ++ ++} ++ ++#if HAVE_MBRTOWC ++static int ++different_multi (const char *old, const char *new, size_t oldlen, size_t newlen, mbstate_t oldstate, mbstate_t newstate) ++{ ++ size_t i, j, chars; ++ const char *str[2]; ++ char *copy[2]; ++ size_t len[2]; ++ mbstate_t state[2]; ++ size_t mblength; ++ wchar_t wc, uwc; ++ mbstate_t state_bak; ++ ++ str[0] = old; ++ str[1] = new; ++ len[0] = oldlen; ++ len[1] = newlen; ++ state[0] = oldstate; ++ state[1] = newstate; ++ ++ for (i = 0; i < 2; i++) ++ { ++ copy[i] = xmalloc (len[i] + 1); ++ memset (copy[i], '\0', len[i] + 1); ++ ++ for (j = 0, chars = 0; j < len[i] && chars < check_chars; chars++) ++ { ++ state_bak = state[i]; ++ mblength = mbrtowc (&wc, str[i] + j, len[i] - j, &(state[i])); ++ ++ switch (mblength) ++ { ++ case (size_t)-1: ++ case (size_t)-2: ++ state[i] = state_bak; ++ /* Fall through */ ++ case 0: ++ mblength = 1; ++ break; ++ ++ default: ++ if (ignore_case) ++ { ++ uwc = towupper (wc); ++ ++ if (uwc != wc) ++ { ++ mbstate_t state_wc; ++ size_t mblen; ++ ++ memset (&state_wc, '\0', sizeof(mbstate_t)); ++ mblen = wcrtomb (copy[i] + j, uwc, &state_wc); ++ assert (mblen != (size_t)-1); ++ } ++ else ++ memcpy (copy[i] + j, str[i] + j, mblength); ++ } ++ else ++ memcpy (copy[i] + j, str[i] + j, mblength); ++ } ++ j += mblength; ++ } ++ copy[i][j] = '\0'; ++ len[i] = j; ++ } ++ int rc = xmemcoll (copy[0], len[0], copy[1], len[1]); ++ free (copy[0]); ++ free (copy[1]); ++ return rc; ++ + } ++#endif + + /* Output the line in linebuffer LINE to standard output + provided that the switches say it should be output. +@@ -367,19 +562,38 @@ check_file (const char *infile, const ch + char *prevfield IF_LINT ( = NULL); + size_t prevlen IF_LINT ( = 0); + bool first_group_printed = false; ++#if HAVE_MBRTOWC ++ mbstate_t prevstate; ++ ++ memset (&prevstate, '\0', sizeof (mbstate_t)); ++#endif + + while (!feof (stdin)) + { + char *thisfield; + size_t thislen; + bool new_group; ++#if HAVE_MBRTOWC ++ mbstate_t thisstate; ++#endif + + if (readlinebuffer_delim (thisline, stdin, delimiter) == 0) + break; + + thisfield = find_field (thisline); + thislen = thisline->length - 1 - (thisfield - thisline->buffer); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ thisstate = thisline->state; + ++ new_group = (prevline->length == 0 ++ || different_multi (thisfield, prevfield, ++ thislen, prevlen, ++ thisstate, prevstate)); ++ } ++ else ++#endif + new_group = (prevline->length == 0 + || different (thisfield, prevfield, thislen, prevlen)); + +@@ -397,6 +611,10 @@ check_file (const char *infile, const ch + SWAP_LINES (prevline, thisline); + prevfield = thisfield; + prevlen = thislen; ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ prevstate = thisstate; ++#endif + first_group_printed = true; + } + } +@@ -409,17 +627,26 @@ check_file (const char *infile, const ch + size_t prevlen; + uintmax_t match_count = 0; + bool first_delimiter = true; ++#if HAVE_MBRTOWC ++ mbstate_t prevstate; ++#endif + + if (readlinebuffer_delim (prevline, stdin, delimiter) == 0) + goto closefiles; + prevfield = find_field (prevline); + prevlen = prevline->length - 1 - (prevfield - prevline->buffer); ++#if HAVE_MBRTOWC ++ prevstate = prevline->state; ++#endif + + while (!feof (stdin)) + { + bool match; + char *thisfield; + size_t thislen; ++#if HAVE_MBRTOWC ++ mbstate_t thisstate = thisline->state; ++#endif + if (readlinebuffer_delim (thisline, stdin, delimiter) == 0) + { + if (ferror (stdin)) +@@ -428,6 +655,14 @@ check_file (const char *infile, const ch + } + thisfield = find_field (thisline); + thislen = thisline->length - 1 - (thisfield - thisline->buffer); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ match = !different_multi (thisfield, prevfield, ++ thislen, prevlen, thisstate, prevstate); ++ } ++ else ++#endif + match = !different (thisfield, prevfield, thislen, prevlen); + match_count += match; + +@@ -460,6 +695,9 @@ check_file (const char *infile, const ch + SWAP_LINES (prevline, thisline); + prevfield = thisfield; + prevlen = thislen; ++#if HAVE_MBRTOWC ++ prevstate = thisstate; ++#endif + if (!match) + match_count = 0; + } +@@ -506,6 +744,19 @@ main (int argc, char **argv) + + atexit (close_stdout); + ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ find_field = find_field_multi; ++ } ++ else ++#endif ++ { ++ find_field = find_field_uni; ++ } ++ ++ ++ + skip_chars = 0; + skip_fields = 0; + check_chars = SIZE_MAX; +diff -Naurp coreutils-8.27-orig/tests/expand/mb.sh coreutils-8.27/tests/expand/mb.sh +--- coreutils-8.27-orig/tests/expand/mb.sh 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/tests/expand/mb.sh 2017-03-11 23:49:06.759133489 -0600 +@@ -0,0 +1,183 @@ ++#!/bin/sh ++ ++# Copyright (C) 2012-2017 Free Software Foundation, Inc. ++ ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the GNU General Public License as published by ++# the Free Software Foundation, either version 3 of the License, or ++# (at your option) any later version. ++ ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++ ++# You should have received a copy of the GNU General Public License ++# along with this program. If not, see . ++ ++. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src ++print_ver_ expand ++ ++export LC_ALL=en_US.UTF-8 ++ ++#input containing multibyte characters ++cat <<\EOF > in || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++EOF ++env printf ' äöü\t. öüä. \tä xx\n' >> in || framework_failure_ ++ ++cat <<\EOF > exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#multiple files as an input ++cat <<\EOF >> exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++expand ./in ./in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#test characters with display widths != 1 ++env printf '12345678 ++e\t|ascii(1) ++\u00E9\t|composed(1) ++e\u0301\t|decomposed(1) ++\u3000\t|ideo-space(2) ++\uFF0D\t|full-hypen(2) ++' > in || framework_failure_ ++ ++env printf '12345678 ++e |ascii(1) ++\u00E9 |composed(1) ++e\u0301 |decomposed(1) ++\u3000 |ideo-space(2) ++\uFF0D |full-hypen(2) ++' > exp || framework_failure_ ++ ++expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#shouldn't fail with "input line too long" ++#when a line starts with a control character ++env printf '\n' > in || framework_failure_ ++ ++expand < in > out || fail=1 ++compare in out > /dev/null 2>&1 || fail=1 ++ ++#non-Unicode characters interspersed between Unicode ones ++env printf '12345678 ++\t\xFF| ++\xFF\t| ++\t\xFFä| ++ä\xFF\t| ++\tä\xFF| ++\xFF\tä| ++äbcdef\xFF\t| ++' > in || framework_failure_ ++ ++env printf '12345678 ++ \xFF| ++\xFF | ++ \xFFä| ++ä\xFF | ++ ä\xFF| ++\xFF ä| ++äbcdef\xFF | ++' > exp || framework_failure_ ++ ++expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++ ++ ++#BOM header test 1 ++printf "\xEF\xBB\xBF" > in; cat <<\EOF >> in || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++EOF ++env printf ' äöü\t. öüä. \tä xx\n' >> in || framework_failure_ ++ ++printf "\xEF\xBB\xBF" > exp; cat <<\EOF >> exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++ ++expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LANG=C expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LC_ALL=C expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++ ++printf '\xEF\xBB\xBF' > in1; cat <<\EOF >> in1 || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++EOF ++env printf ' äöü\t. öüä. \tä xx\n' >> in1 || framework_failure_ ++ ++ ++printf '\xEF\xBB\xBF' > exp; cat <<\EOF >> exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++expand in1 in1 > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LANG=C expand in1 in1 > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LC_ALL=C expand in1 in1 > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++exit $fail +diff -Naurp coreutils-8.27-orig/tests/i18n/sort.sh coreutils-8.27/tests/i18n/sort.sh +--- coreutils-8.27-orig/tests/i18n/sort.sh 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/tests/i18n/sort.sh 2017-03-11 23:47:13.100285838 -0600 +@@ -0,0 +1,29 @@ ++#!/bin/sh ++# Verify sort's multi-byte support. ++ ++. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src ++print_ver_ sort ++ ++export LC_ALL=en_US.UTF-8 ++locale -k LC_CTYPE | grep -q "charmap.*UTF-8" \ ++ || skip_ "No UTF-8 locale available" ++ ++# Enable heap consistency checkng on older systems ++export MALLOC_CHECK_=2 ++ ++ ++# check buffer overflow issue due to ++# expanding multi-byte representation due to case conversion ++# https://bugzilla.suse.com/show_bug.cgi?id=928749 ++cat < exp ++. ++ɑ ++EOF ++cat < out || fail=1 ++. ++ɑ ++EOF ++compare exp out || { fail=1; cat out; } ++ ++ ++Exit $fail +diff -Naurp coreutils-8.27-orig/tests/local.mk coreutils-8.27/tests/local.mk +--- coreutils-8.27-orig/tests/local.mk 2017-02-28 22:25:37.000000000 -0600 ++++ coreutils-8.27/tests/local.mk 2017-03-11 23:47:38.072058253 -0600 +@@ -352,6 +352,8 @@ all_tests = \ + tests/misc/sort-discrim.sh \ + tests/misc/sort-files0-from.pl \ + tests/misc/sort-float.sh \ ++ tests/misc/sort-mb-tests.sh \ ++ tests/i18n/sort.sh \ + tests/misc/sort-h-thousands-sep.sh \ + tests/misc/sort-merge.pl \ + tests/misc/sort-merge-fdlimit.sh \ +@@ -544,6 +546,7 @@ all_tests = \ + tests/du/threshold.sh \ + tests/du/trailing-slash.sh \ + tests/du/two-args.sh \ ++ tests/expand/mb.sh \ + tests/id/gnu-zero-uids.sh \ + tests/id/no-context.sh \ + tests/id/context.sh \ +@@ -684,6 +687,7 @@ all_tests = \ + tests/touch/read-only.sh \ + tests/touch/relative.sh \ + tests/touch/trailing-slash.sh \ ++ tests/unexpand/mb.sh \ + $(all_root_tests) + + # See tests/factor/create-test.sh. +diff -Naurp coreutils-8.27-orig/tests/misc/cut.pl coreutils-8.27/tests/misc/cut.pl +--- coreutils-8.27-orig/tests/misc/cut.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/cut.pl 2017-03-11 23:47:13.100285838 -0600 +@@ -23,9 +23,11 @@ use strict; + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + +-my $mb_locale = $ENV{LOCALE_FR_UTF8}; ++my $mb_locale; ++# uncommented enable multibyte paths ++$mb_locale = $ENV{LOCALE_FR_UTF8}; + ! defined $mb_locale || $mb_locale eq 'none' +- and $mb_locale = 'C'; ++ and $mb_locale = 'C'; + + my $prog = 'cut'; + my $try = "Try '$prog --help' for more information.\n"; +@@ -240,6 +242,7 @@ if ($mb_locale ne 'C') + my @new_t = @$t; + my $test_name = shift @new_t; + ++ next if ($test_name =~ "newline-[12][0-9]"); + push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; + } + push @Tests, @new; +diff -Naurp coreutils-8.27-orig/tests/misc/expand.pl coreutils-8.27/tests/misc/expand.pl +--- coreutils-8.27-orig/tests/misc/expand.pl 2017-03-01 11:16:46.000000000 -0600 ++++ coreutils-8.27/tests/misc/expand.pl 2017-03-11 23:47:13.101285788 -0600 +@@ -27,6 +27,15 @@ my $prog = 'expand'; + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + ++#comment out next line to disable multibyte tests ++my $mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ ++my $prog = 'expand'; ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + my @Tests = + ( + ['t1', '--tabs=3', {IN=>"a\tb"}, {OUT=>"a b"}], +@@ -152,6 +161,8 @@ my @Tests = + ['trail9', '--tab=1,2 -t/5',{IN=>"\ta\tb\tc"}, {OUT=>" a b c"}], + + # Test errors ++ # FIXME: The following tests contain ‘quoting’ specific to LC_MESSAGES ++ # So we force LC_MESSAGES=C to make them pass. + ['e1', '--tabs="a"', {IN=>''}, {OUT=>''}, {EXIT=>1}, + {ERR => "$prog: tab size contains invalid character(s): 'a'\n"}], + ['e2', "-t $UINTMAX_OFLOW", {IN=>''}, {OUT=>''}, {EXIT=>1}, +@@ -168,6 +179,37 @@ my @Tests = + {ERR => "$prog: '/' specifier not at start of number: '/'\n"}], + ); + ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether expand is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ push @new, ["$test_name-mb", @new_t, {ENV => "LANG=$mb_locale LC_MESSAGES=C"}]; ++ } ++ push @Tests, @new; ++ } ++ ++ ++@Tests = triple_test \@Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +diff -Naurp coreutils-8.27-orig/tests/misc/fold.pl coreutils-8.27/tests/misc/fold.pl +--- coreutils-8.27-orig/tests/misc/fold.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/fold.pl 2017-03-11 23:47:13.101285788 -0600 +@@ -20,9 +20,18 @@ use strict; + + (my $program_name = $0) =~ s|.*/||; + ++my $prog = 'fold'; ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + ++# uncommented to enable multibyte paths ++my $mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ + my @Tests = + ( + ['s1', '-w2 -s', {IN=>"a\t"}, {OUT=>"a\n\t"}], +@@ -31,9 +40,48 @@ my @Tests = + ['s4', '-w4 -s', {IN=>"abc ef\n"}, {OUT=>"abc \nef\n"}], + ); + ++# Add _POSIX2_VERSION=199209 to the environment of each test ++# that uses an old-style option like +1. ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether fold is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ ++@Tests = triple_test \@Tests; ++ ++# Remember that triple_test creates from each test with exactly one "IN" ++# file two more tests (.p and .r suffix on name) corresponding to reading ++# input from a file and from a pipe. The pipe-reading test would fail ++# due to a race condition about 1 in 20 times. ++# Remove the IN_PIPE version of the "output-is-input" test above. ++# The others aren't susceptible because they have three inputs each. ++@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +-my $prog = 'fold'; + my $fail = run_tests ($program_name, $prog, \@Tests, $save_temps, $verbose); + exit $fail; +diff -Naurp coreutils-8.27-orig/tests/misc/join.pl coreutils-8.27/tests/misc/join.pl +--- coreutils-8.27-orig/tests/misc/join.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/join.pl 2017-03-11 23:47:13.102285737 -0600 +@@ -25,6 +25,15 @@ my $limits = getlimits (); + + my $prog = 'join'; + ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ ++my $mb_locale; ++#Comment out next line to disable multibyte tests ++$mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ + my $delim = chr 0247; + sub t_subst ($) + { +@@ -329,8 +338,49 @@ foreach my $t (@tv) + push @Tests, $new_ent; + } + ++# Add _POSIX2_VERSION=199209 to the environment of each test ++# that uses an old-style option like +1. ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether join is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ #Adjust the output some error messages including test_name for mb ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR}} ++ (@new_t)) ++ { ++ my $sub2 = {ERR_SUBST => "s/$test_name-mb/$test_name/"}; ++ push @new_t, $sub2; ++ push @$t, $sub2; ++ } ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ + @Tests = triple_test \@Tests; + ++#skip invalid-j-mb test, it is failing because of the format ++@Tests = grep {$_->[0] ne 'invalid-j-mb'} @Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +diff -Naurp coreutils-8.27-orig/tests/misc/sort-mb-tests.sh coreutils-8.27/tests/misc/sort-mb-tests.sh +--- coreutils-8.27-orig/tests/misc/sort-mb-tests.sh 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/tests/misc/sort-mb-tests.sh 2017-03-11 23:47:13.102285737 -0600 +@@ -0,0 +1,45 @@ ++#!/bin/sh ++# Verify sort's multi-byte support. ++ ++. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src ++print_ver_ sort ++ ++export LC_ALL=en_US.UTF-8 ++locale -k LC_CTYPE | grep -q "charmap.*UTF-8" \ ++ || skip_ "No UTF-8 locale available" ++ ++ ++cat < exp ++Banana＠5 ++Apple＠10 ++Citrus＠20 ++Cherry＠30 ++EOF ++ ++cat < out || fail=1 ++Apple＠10 ++Banana＠5 ++Citrus＠20 ++Cherry＠30 ++EOF ++ ++compare exp out || { fail=1; cat out; } ++ ++ ++cat < exp ++Citrus＠ＡＡ20＠＠5 ++Cherry＠ＡＡ30＠＠10 ++Apple＠ＡＡ10＠＠20 ++Banana＠ＡＡ5＠＠30 ++EOF ++ ++cat < out || fail=1 ++Apple＠ＡＡ10＠＠20 ++Banana＠ＡＡ5＠＠30 ++Citrus＠ＡＡ20＠＠5 ++Cherry＠ＡＡ30＠＠10 ++EOF ++ ++compare exp out || { fail=1; cat out; } ++ ++Exit $fail +diff -Naurp coreutils-8.27-orig/tests/misc/sort-merge.pl coreutils-8.27/tests/misc/sort-merge.pl +--- coreutils-8.27-orig/tests/misc/sort-merge.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/sort-merge.pl 2017-03-11 23:47:13.102285737 -0600 +@@ -26,6 +26,15 @@ my $prog = 'sort'; + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + ++my $mb_locale; ++# uncommented according to upstream commit enabling multibyte paths ++$mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + # three empty files and one that says 'foo' + my @inputs = (+(map{{IN=> {"empty$_"=> ''}}}1..3), {IN=> {foo=> "foo\n"}}); + +@@ -77,6 +86,39 @@ my @Tests = + {OUT=>$big_input}], + ); + ++# Add _POSIX2_VERSION=199209 to the environment of each test ++# that uses an old-style option like +1. ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether sort is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ next if ($test_name =~ "nmerge-."); ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ ++@Tests = triple_test \@Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +diff -Naurp coreutils-8.27-orig/tests/misc/sort.pl coreutils-8.27/tests/misc/sort.pl +--- coreutils-8.27-orig/tests/misc/sort.pl 2017-01-21 08:53:43.000000000 -0600 ++++ coreutils-8.27/tests/misc/sort.pl 2017-03-11 23:47:13.103285687 -0600 +@@ -24,10 +24,15 @@ my $prog = 'sort'; + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + +-my $mb_locale = $ENV{LOCALE_FR_UTF8}; ++my $mb_locale; ++#Comment out next line to disable multibyte tests ++$mb_locale = $ENV{LOCALE_FR_UTF8}; + ! defined $mb_locale || $mb_locale eq 'none' + and $mb_locale = 'C'; + ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + # Since each test is run with a file name and with redirected stdin, + # the name in the diagnostic is either the file name or "-". + # Normalize each diagnostic to use '-'. +@@ -423,6 +428,38 @@ foreach my $t (@Tests) + } + } + ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether sort is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ #disable several failing tests until investigation, disable all tests with envvars set ++ next if (grep {ref $_ eq 'HASH' && exists $_->{ENV}} (@new_t)); ++ next if ($test_name =~ "18g" or $test_name =~ "sort-numeric" or $test_name =~ "08[ab]" or $test_name =~ "03[def]" or $test_name =~ "h4" or $test_name =~ "n1" or $test_name =~ "2[01]a"); ++ next if ($test_name =~ "11[ab]"); # avoid FP: expected result differs to MB result due to collation rules. ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ + @Tests = triple_test \@Tests; + + # Remember that triple_test creates from each test with exactly one "IN" +@@ -432,6 +469,7 @@ foreach my $t (@Tests) + # Remove the IN_PIPE version of the "output-is-input" test above. + # The others aren't susceptible because they have three inputs each. + @Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; ++@Tests = grep {$_->[0] ne 'output-is-input-mb.p'} @Tests; + + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; +diff -Naurp coreutils-8.27-orig/tests/misc/unexpand.pl coreutils-8.27/tests/misc/unexpand.pl +--- coreutils-8.27-orig/tests/misc/unexpand.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/unexpand.pl 2017-03-11 23:47:13.103285687 -0600 +@@ -27,6 +27,14 @@ my $limits = getlimits (); + + my $prog = 'unexpand'; + ++# comment out next line to disable multibyte tests ++my $mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + my @Tests = + ( + ['a1', {IN=> ' 'x 1 ."y\n"}, {OUT=> ' 'x 1 ."y\n"}], +@@ -128,6 +136,37 @@ my @Tests = + ['ts2', '-t5,8', {IN=>"x\t \t y\n"}, {OUT=>"x\t\t y\n"}], + ); + ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether unexpand is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ next if ($test_name =~ 'b-1'); ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ ++@Tests = triple_test \@Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +diff -Naurp coreutils-8.27-orig/tests/misc/uniq.pl coreutils-8.27/tests/misc/uniq.pl +--- coreutils-8.27-orig/tests/misc/uniq.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/uniq.pl 2017-03-11 23:47:13.103285687 -0600 +@@ -23,9 +23,17 @@ my $limits = getlimits (); + my $prog = 'uniq'; + my $try = "Try '$prog --help' for more information.\n"; + ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + ++my $mb_locale; ++#Comment out next line to disable multibyte tests ++$mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ + # When possible, create a "-z"-testing variant of each test. + sub add_z_variants($) + { +@@ -262,6 +270,53 @@ foreach my $t (@Tests) + and push @$t, {ENV=>'_POSIX2_VERSION=199209'}; + } + ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether uniq is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ # In test #145, replace the each ‘...’ by '...'. ++ if ($test_name =~ "145") ++ { ++ my $sub = { ERR_SUBST => "s/‘([^’]+)’/'\$1'/g"}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ next if ( $test_name =~ "schar" ++ or $test_name =~ "^obs-plus" ++ or $test_name =~ "119"); ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ ++# Remember that triple_test creates from each test with exactly one "IN" ++# file two more tests (.p and .r suffix on name) corresponding to reading ++# input from a file and from a pipe. The pipe-reading test would fail ++# due to a race condition about 1 in 20 times. ++# Remove the IN_PIPE version of the "output-is-input" test above. ++# The others aren't susceptible because they have three inputs each. ++ ++@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; ++ + @Tests = add_z_variants \@Tests; + @Tests = triple_test \@Tests; + +diff -Naurp coreutils-8.27-orig/tests/pr/pr-tests.pl coreutils-8.27/tests/pr/pr-tests.pl +--- coreutils-8.27-orig/tests/pr/pr-tests.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/pr/pr-tests.pl 2017-03-11 23:47:13.103285687 -0600 +@@ -24,6 +24,15 @@ use strict; + my $prog = 'pr'; + my $normalize_strerror = "s/': .*/'/"; + ++my $mb_locale; ++#Uncomment the following line to enable multibyte tests ++$mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + my @tv = ( + + # -b option is no longer an official option. But it's still working to +@@ -474,8 +483,48 @@ push @Tests, + {IN=>{2=>"a\n"}}, + {OUT=>"a\t\t\t\t \t\t\ta\n"} ]; + ++# Add _POSIX2_VERSION=199209 to the environment of each test ++# that uses an old-style option like +1. ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether pr is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ #temporarily skip some failing tests ++ next if ($test_name =~ "col-0" or $test_name =~ "col-inval" or $test_name =~ "asan1"); ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ + @Tests = triple_test \@Tests; + ++# Remember that triple_test creates from each test with exactly one "IN" ++# file two more tests (.p and .r suffix on name) corresponding to reading ++# input from a file and from a pipe. The pipe-reading test would fail ++# due to a race condition about 1 in 20 times. ++# Remove the IN_PIPE version of the "output-is-input" test above. ++# The others aren't susceptible because they have three inputs each. ++@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +diff -Naurp coreutils-8.27-orig/tests/unexpand/mb.sh coreutils-8.27/tests/unexpand/mb.sh +--- coreutils-8.27-orig/tests/unexpand/mb.sh 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/tests/unexpand/mb.sh 2017-03-11 23:49:06.759133489 -0600 +@@ -0,0 +1,172 @@ ++#!/bin/sh ++ ++# Copyright (C) 2012-2017 Free Software Foundation, Inc. ++ ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the GNU General Public License as published by ++# the Free Software Foundation, either version 3 of the License, or ++# (at your option) any later version. ++ ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++ ++# You should have received a copy of the GNU General Public License ++# along with this program. If not, see . ++ ++. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src ++print_ver_ unexpand ++ ++export LC_ALL=en_US.UTF-8 ++ ++#input containing multibyte characters ++cat > in <<\EOF ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++cat > exp <<\EOF ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++unexpand -a < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++ ++#multiple files as an input ++cat >> exp <<\EOF ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++ ++unexpand -a ./in ./in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#test characters with a display width larger than 1 ++ ++env printf '12345678 ++e |ascii(1) ++\u00E9 |composed(1) ++e\u0301 |decomposed(1) ++\u3000 |ideo-space(2) ++\uFF0D |full-hypen(2) ++' > in || framework_failure_ ++ ++env printf '12345678 ++e\t|ascii(1) ++\u00E9\t|composed(1) ++e\u0301\t|decomposed(1) ++\u3000\t|ideo-space(2) ++\uFF0D\t|full-hypen(2) ++' > exp || framework_failure_ ++ ++unexpand -a < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#test input where a blank of width > 1 is not being substituted ++in="$(LC_ALL=en_US.UTF-8 printf ' \u3000 ö ü ß')" ++exp=' 　 ö ü ß' ++ ++unexpand -a < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#non-Unicode characters interspersed between Unicode ones ++env printf '12345678 ++ \xFF| ++\xFF | ++ \xFFä| ++ä\xFF | ++ ä\xFF| ++\xFF ä| ++äbcdef\xFF | ++' > in || framework_failure_ ++ ++env printf '12345678 ++\t\xFF| ++\xFF\t| ++\t\xFFä| ++ä\xFF\t| ++\tä\xFF| ++\xFF\tä| ++äbcdef\xFF\t| ++' > exp || framework_failure_ ++ ++unexpand -a < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#BOM header test 1 ++printf "\xEF\xBB\xBF" > in; cat <<\EOF >> in || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++env printf ' äöü\t. öüä. \tä xx\n' >> in || framework_failure_ ++ ++printf "\xEF\xBB\xBF" > exp; cat <<\EOF >> exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++unexpand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LANG=C unexpand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LC_ALL=C unexpand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++ ++printf "\xEF\xBB\xBF" > exp; cat <<\EOF >> exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++ ++unexpand in in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LANG=C unexpand in in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LC_ALL=C unexpand in in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 diff -r c3c324c07d3f -r 90a5eb560fd6 coreutils/stuff/patches/coreutils-8.29-i18n-1.patch --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/coreutils/stuff/patches/coreutils-8.29-i18n-1.patch Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,5521 @@ +Submitted by: DJ Lucas (dj_AT_linuxfromscratch_DOT_org) +Date: 2017-03-12 +Initial Package Version: 8.27 +Upstream Status: Rejected +Origin: Based on Fedora's i18n patches at + http://pkgs.fedoraproject.org/cgit/rpms/coreutils.git/tree/ +Description: Fixes i18n issues with various Coreutils programs + +diff -Naurp coreutils-8.27-orig/bootstrap.conf coreutils-8.27/bootstrap.conf +--- coreutils-8.27-orig/bootstrap.conf 2017-03-07 23:34:06.000000000 -0600 ++++ coreutils-8.27/bootstrap.conf 2017-03-11 23:47:38.068058445 -0600 +@@ -152,6 +152,7 @@ gnulib_modules=" + maintainer-makefile + malloc-gnu + manywarnings ++ mbfile + mbrlen + mbrtowc + mbsalign +diff -Naurp coreutils-8.27-orig/configure.ac coreutils-8.27/configure.ac +--- coreutils-8.27-orig/configure.ac 2017-02-26 08:52:29.000000000 -0600 ++++ coreutils-8.27/configure.ac 2017-03-11 23:47:38.068058445 -0600 +@@ -429,6 +429,8 @@ fi + # I'm leaving it here for now. This whole thing needs to be modernized... + gl_WINSIZE_IN_PTEM + ++gl_MBFILE ++ + gl_HEADER_TIOCGWINSZ_IN_TERMIOS_H + + if test $gl_cv_sys_tiocgwinsz_needs_termios_h = no && \ +diff -Naurp coreutils-8.27-orig/lib/linebuffer.h coreutils-8.27/lib/linebuffer.h +--- coreutils-8.27-orig/lib/linebuffer.h 2017-01-01 16:35:38.000000000 -0600 ++++ coreutils-8.27/lib/linebuffer.h 2017-03-11 23:47:13.089286391 -0600 +@@ -21,6 +21,11 @@ + + # include + ++/* Get mbstate_t. */ ++# if HAVE_WCHAR_H ++# include ++# endif ++ + /* A 'struct linebuffer' holds a line of text. */ + + struct linebuffer +@@ -28,6 +33,9 @@ struct linebuffer + size_t size; /* Allocated. */ + size_t length; /* Used. */ + char *buffer; ++# if HAVE_WCHAR_H ++ mbstate_t state; ++# endif + }; + + /* Initialize linebuffer LINEBUFFER for use. */ +diff -Naurp coreutils-8.27-orig/lib/mbfile.c coreutils-8.27/lib/mbfile.c +--- coreutils-8.27-orig/lib/mbfile.c 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/lib/mbfile.c 2017-03-11 23:47:38.069058397 -0600 +@@ -0,0 +1,3 @@ ++#include ++#define MBFILE_INLINE _GL_EXTERN_INLINE ++#include "mbfile.h" +diff -Naurp coreutils-8.27-orig/lib/mbfile.h coreutils-8.27/lib/mbfile.h +--- coreutils-8.27-orig/lib/mbfile.h 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/lib/mbfile.h 2017-03-11 23:47:38.069058397 -0600 +@@ -0,0 +1,255 @@ ++/* Multibyte character I/O: macros for multi-byte encodings. ++ Copyright (C) 2001, 2005, 2009-2017 Free Software Foundation, Inc. ++ ++ This program is free software: you can redistribute it and/or modify ++ it under the terms of the GNU General Public License as published by ++ the Free Software Foundation; either version 3 of the License, or ++ (at your option) any later version. ++ ++ This program is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ GNU General Public License for more details. ++ ++ You should have received a copy of the GNU General Public License ++ along with this program. If not, see . */ ++ ++/* Written by Mitsuru Chinen ++ and Bruno Haible . */ ++ ++/* The macros in this file implement multi-byte character input from a ++ stream. ++ ++ mb_file_t ++ is the type for multibyte character input stream, usable for variable ++ declarations. ++ ++ mbf_char_t ++ is the type for multibyte character or EOF, usable for variable ++ declarations. ++ ++ mbf_init (mbf, stream) ++ initializes the MB_FILE for reading from stream. ++ ++ mbf_getc (mbc, mbf) ++ reads the next multibyte character from mbf and stores it in mbc. ++ ++ mb_iseof (mbc) ++ returns true if mbc represents the EOF value. ++ ++ Here are the function prototypes of the macros. ++ ++ extern void mbf_init (mb_file_t mbf, FILE *stream); ++ extern void mbf_getc (mbf_char_t mbc, mb_file_t mbf); ++ extern bool mb_iseof (const mbf_char_t mbc); ++ */ ++ ++#ifndef _MBFILE_H ++#define _MBFILE_H 1 ++ ++#include ++#include ++#include ++#include ++ ++/* Tru64 with Desktop Toolkit C has a bug: must be included before ++ . ++ BSD/OS 4.1 has a bug: and must be included before ++ . */ ++#include ++#include ++#include ++ ++#include "mbchar.h" ++ ++#ifndef _GL_INLINE_HEADER_BEGIN ++ #error "Please include config.h first." ++#endif ++_GL_INLINE_HEADER_BEGIN ++#ifndef MBFILE_INLINE ++# define MBFILE_INLINE _GL_INLINE ++#endif ++ ++struct mbfile_multi { ++ FILE *fp; ++ bool eof_seen; ++ bool have_pushback; ++ mbstate_t state; ++ unsigned int bufcount; ++ char buf[MBCHAR_BUF_SIZE]; ++ struct mbchar pushback; ++}; ++ ++MBFILE_INLINE void ++mbfile_multi_getc (struct mbchar *mbc, struct mbfile_multi *mbf) ++{ ++ size_t bytes; ++ ++ /* If EOF has already been seen, don't use getc. This matters if ++ mbf->fp is connected to an interactive tty. */ ++ if (mbf->eof_seen) ++ goto eof; ++ ++ /* Return character pushed back, if there is one. */ ++ if (mbf->have_pushback) ++ { ++ mb_copy (mbc, &mbf->pushback); ++ mbf->have_pushback = false; ++ return; ++ } ++ ++ /* Before using mbrtowc, we need at least one byte. */ ++ if (mbf->bufcount == 0) ++ { ++ int c = getc (mbf->fp); ++ if (c == EOF) ++ { ++ mbf->eof_seen = true; ++ goto eof; ++ } ++ mbf->buf[0] = (unsigned char) c; ++ mbf->bufcount++; ++ } ++ ++ /* Handle most ASCII characters quickly, without calling mbrtowc(). */ ++ if (mbf->bufcount == 1 && mbsinit (&mbf->state) && is_basic (mbf->buf[0])) ++ { ++ /* These characters are part of the basic character set. ISO C 99 ++ guarantees that their wide character code is identical to their ++ char code. */ ++ mbc->wc = mbc->buf[0] = mbf->buf[0]; ++ mbc->wc_valid = true; ++ mbc->ptr = &mbc->buf[0]; ++ mbc->bytes = 1; ++ mbf->bufcount = 0; ++ return; ++ } ++ ++ /* Use mbrtowc on an increasing number of bytes. Read only as many bytes ++ from mbf->fp as needed. This is needed to give reasonable interactive ++ behaviour when mbf->fp is connected to an interactive tty. */ ++ for (;;) ++ { ++ /* We don't know whether the 'mbrtowc' function updates the state when ++ it returns -2, - this is the ISO C 99 and glibc-2.2 behaviour - or ++ not - amended ANSI C, glibc-2.1 and Solaris 2.7 behaviour. We ++ don't have an autoconf test for this, yet. ++ The new behaviour would allow us to feed the bytes one by one into ++ mbrtowc. But the old behaviour forces us to feed all bytes since ++ the end of the last character into mbrtowc. Since we want to retry ++ with more bytes when mbrtowc returns -2, we must backup the state ++ before calling mbrtowc, because implementations with the new ++ behaviour will clobber it. */ ++ mbstate_t backup_state = mbf->state; ++ ++ bytes = mbrtowc (&mbc->wc, &mbf->buf[0], mbf->bufcount, &mbf->state); ++ ++ if (bytes == (size_t) -1) ++ { ++ /* An invalid multibyte sequence was encountered. */ ++ /* Return a single byte. */ ++ bytes = 1; ++ mbc->wc_valid = false; ++ break; ++ } ++ else if (bytes == (size_t) -2) ++ { ++ /* An incomplete multibyte character. */ ++ mbf->state = backup_state; ++ if (mbf->bufcount == MBCHAR_BUF_SIZE) ++ { ++ /* An overlong incomplete multibyte sequence was encountered. */ ++ /* Return a single byte. */ ++ bytes = 1; ++ mbc->wc_valid = false; ++ break; ++ } ++ else ++ { ++ /* Read one more byte and retry mbrtowc. */ ++ int c = getc (mbf->fp); ++ if (c == EOF) ++ { ++ /* An incomplete multibyte character at the end. */ ++ mbf->eof_seen = true; ++ bytes = mbf->bufcount; ++ mbc->wc_valid = false; ++ break; ++ } ++ mbf->buf[mbf->bufcount] = (unsigned char) c; ++ mbf->bufcount++; ++ } ++ } ++ else ++ { ++ if (bytes == 0) ++ { ++ /* A null wide character was encountered. */ ++ bytes = 1; ++ assert (mbf->buf[0] == '\0'); ++ assert (mbc->wc == 0); ++ } ++ mbc->wc_valid = true; ++ break; ++ } ++ } ++ ++ /* Return the multibyte sequence mbf->buf[0..bytes-1]. */ ++ mbc->ptr = &mbc->buf[0]; ++ memcpy (&mbc->buf[0], &mbf->buf[0], bytes); ++ mbc->bytes = bytes; ++ ++ mbf->bufcount -= bytes; ++ if (mbf->bufcount > 0) ++ { ++ /* It's not worth calling memmove() for so few bytes. */ ++ unsigned int count = mbf->bufcount; ++ char *p = &mbf->buf[0]; ++ ++ do ++ { ++ *p = *(p + bytes); ++ p++; ++ } ++ while (--count > 0); ++ } ++ return; ++ ++eof: ++ /* An mbchar_t with bytes == 0 is used to indicate EOF. */ ++ mbc->ptr = NULL; ++ mbc->bytes = 0; ++ mbc->wc_valid = false; ++ return; ++} ++ ++MBFILE_INLINE void ++mbfile_multi_ungetc (const struct mbchar *mbc, struct mbfile_multi *mbf) ++{ ++ mb_copy (&mbf->pushback, mbc); ++ mbf->have_pushback = true; ++} ++ ++typedef struct mbfile_multi mb_file_t; ++ ++typedef mbchar_t mbf_char_t; ++ ++#define mbf_init(mbf, stream) \ ++ ((mbf).fp = (stream), \ ++ (mbf).eof_seen = false, \ ++ (mbf).have_pushback = false, \ ++ memset (&(mbf).state, '\0', sizeof (mbstate_t)), \ ++ (mbf).bufcount = 0) ++ ++#define mbf_getc(mbc, mbf) mbfile_multi_getc (&(mbc), &(mbf)) ++ ++#define mbf_ungetc(mbc, mbf) mbfile_multi_ungetc (&(mbc), &(mbf)) ++ ++#define mb_iseof(mbc) ((mbc).bytes == 0) ++ ++#ifndef _GL_INLINE_HEADER_BEGIN ++ #error "Please include config.h first." ++#endif ++_GL_INLINE_HEADER_BEGIN ++ ++#endif /* _MBFILE_H */ +diff -Naurp coreutils-8.27-orig/m4/mbfile.m4 coreutils-8.27/m4/mbfile.m4 +--- coreutils-8.27-orig/m4/mbfile.m4 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/m4/mbfile.m4 2017-03-11 23:47:38.070058349 -0600 +@@ -0,0 +1,14 @@ ++# mbfile.m4 serial 7 ++dnl Copyright (C) 2005, 2008-2017 Free Software Foundation, Inc. ++dnl This file is free software; the Free Software Foundation ++dnl gives unlimited permission to copy and/or distribute it, ++dnl with or without modifications, as long as this notice is preserved. ++ ++dnl autoconf tests required for use of mbfile.h ++dnl From Bruno Haible. ++ ++AC_DEFUN([gl_MBFILE], ++[ ++ AC_REQUIRE([AC_TYPE_MBSTATE_T]) ++ : ++]) +diff -Naurp coreutils-8.27-orig/src/cut.c coreutils-8.27/src/cut.c +--- coreutils-8.27-orig/src/cut.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/cut.c 2017-03-11 23:47:59.526048471 -0600 +@@ -28,6 +28,11 @@ + #include + #include + #include ++ ++/* Get mbstate_t, mbrtowc(). */ ++#if HAVE_WCHAR_H ++# include ++#endif + #include "system.h" + + #include "error.h" +@@ -38,6 +43,18 @@ + + #include "set-fields.h" + ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 ++# undef MB_LEN_MAX ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ + /* The official name of this program (e.g., no 'g' prefix). */ + #define PROGRAM_NAME "cut" + +@@ -54,6 +71,52 @@ + } \ + while (0) + ++/* Refill the buffer BUF to get a multibyte character. */ ++#define REFILL_BUFFER(BUF, BUFPOS, BUFLEN, STREAM) \ ++ do \ ++ { \ ++ if (BUFLEN < MB_LEN_MAX && !feof (STREAM) && !ferror (STREAM)) \ ++ { \ ++ memmove (BUF, BUFPOS, BUFLEN); \ ++ BUFLEN += fread (BUF + BUFLEN, sizeof(char), BUFSIZ, STREAM); \ ++ BUFPOS = BUF; \ ++ } \ ++ } \ ++ while (0) ++ ++/* Get wide character on BUFPOS. BUFPOS is not included after that. ++ If byte sequence is not valid as a character, CONVFAIL is true. Otherwise false. */ ++#define GET_NEXT_WC_FROM_BUFFER(WC, BUFPOS, BUFLEN, MBLENGTH, STATE, CONVFAIL) \ ++ do \ ++ { \ ++ mbstate_t state_bak; \ ++ \ ++ if (BUFLEN < 1) \ ++ { \ ++ WC = WEOF; \ ++ break; \ ++ } \ ++ \ ++ /* Get a wide character. */ \ ++ CONVFAIL = false; \ ++ state_bak = STATE; \ ++ MBLENGTH = mbrtowc ((wchar_t *)&WC, BUFPOS, BUFLEN, &STATE); \ ++ \ ++ switch (MBLENGTH) \ ++ { \ ++ case (size_t)-1: \ ++ case (size_t)-2: \ ++ CONVFAIL = true; \ ++ STATE = state_bak; \ ++ /* Fall througn. */ \ ++ \ ++ case 0: \ ++ MBLENGTH = 1; \ ++ break; \ ++ } \ ++ } \ ++ while (0) ++ + + /* Pointer inside RP. When checking if a byte or field is selected + by a finite range, we check if it is between CURRENT_RP.LO +@@ -61,6 +124,9 @@ + CURRENT_RP.HI then we make CURRENT_RP to point to the next range pair. */ + static struct field_range_pair *current_rp; + ++/* Length of the delimiter given as argument to -d. */ ++size_t delimlen; ++ + /* This buffer is used to support the semantics of the -s option + (or lack of same) when the specified field list includes (does + not include) the first field. In both of those cases, the entire +@@ -77,15 +143,25 @@ enum operating_mode + { + undefined_mode, + +- /* Output characters that are in the given bytes. */ ++ /* Output bytes that are at the given positions. */ + byte_mode, + ++ /* Output characters that are at the given positions. */ ++ character_mode, ++ + /* Output the given delimiter-separated fields. */ + field_mode + }; + + static enum operating_mode operating_mode; + ++/* If nonzero, when in byte mode, don't split multibyte characters. */ ++static int byte_mode_character_aware; ++ ++/* If nonzero, the function for single byte locale is work ++ if this program runs on multibyte locale. */ ++static int force_singlebyte_mode; ++ + /* If true do not output lines containing no delimiter characters. + Otherwise, all such lines are printed. This option is valid only + with field mode. */ +@@ -97,6 +173,9 @@ static bool complement; + + /* The delimiter character for field mode. */ + static unsigned char delim; ++#if HAVE_WCHAR_H ++static wchar_t wcdelim; ++#endif + + /* The delimiter for each line/record. */ + static unsigned char line_delim = '\n'; +@@ -164,7 +243,7 @@ Print selected parts of lines from each + -f, --fields=LIST select only these fields; also print any line\n\ + that contains no delimiter character, unless\n\ + the -s option is specified\n\ +- -n (ignored)\n\ ++ -n with -b: don't split multibyte characters\n\ + "), stdout); + fputs (_("\ + --complement complement the set of selected bytes, characters\n\ +@@ -280,6 +359,82 @@ cut_bytes (FILE *stream) + } + } + ++#if HAVE_MBRTOWC ++/* This function is in use for the following case. ++ ++ 1. Read from the stream STREAM, printing to standard output any selected ++ characters. ++ ++ 2. Read from stream STREAM, printing to standard output any selected bytes, ++ without splitting multibyte characters. */ ++ ++static void ++cut_characters_or_cut_bytes_no_split (FILE *stream) ++{ ++ size_t idx; /* number of bytes or characters in the line so far. */ ++ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ ++ char *bufpos; /* Next read position of BUF. */ ++ size_t buflen; /* The length of the byte sequence in buf. */ ++ wint_t wc; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character which shows ++ as same character as WC. */ ++ mbstate_t state; /* State of the stream. */ ++ bool convfail = false; /* true, when conversion failed. Otherwise false. */ ++ /* Whether to begin printing delimiters between ranges for the current line. ++ Set after we've begun printing data corresponding to the first range. */ ++ bool print_delimiter = false; ++ ++ idx = 0; ++ buflen = 0; ++ bufpos = buf; ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ current_rp = frp; ++ ++ while (1) ++ { ++ REFILL_BUFFER (buf, bufpos, buflen, stream); ++ ++ GET_NEXT_WC_FROM_BUFFER (wc, bufpos, buflen, mblength, state, convfail); ++ (void) convfail; /* ignore unused */ ++ ++ if (wc == WEOF) ++ { ++ if (idx > 0) ++ putchar (line_delim); ++ break; ++ } ++ else if (wc == line_delim) ++ { ++ putchar (line_delim); ++ idx = 0; ++ print_delimiter = false; ++ current_rp = frp; ++ } ++ else ++ { ++ next_item (&idx); ++ if (print_kth (idx)) ++ { ++ if (output_delimiter_specified) ++ { ++ if (print_delimiter && is_range_start_index (idx)) ++ { ++ fwrite (output_delimiter_string, sizeof (char), ++ output_delimiter_length, stdout); ++ } ++ print_delimiter = true; ++ } ++ fwrite (bufpos, mblength, sizeof(char), stdout); ++ } ++ } ++ ++ buflen -= mblength; ++ bufpos += mblength; ++ } ++} ++#endif ++ + /* Read from stream STREAM, printing to standard output any selected fields. */ + + static void +@@ -425,13 +580,211 @@ cut_fields (FILE *stream) + } + } + ++#if HAVE_MBRTOWC ++static void ++cut_fields_mb (FILE *stream) ++{ ++ int c; ++ size_t field_idx; ++ int found_any_selected_field; ++ int buffer_first_field; ++ int empty_input; ++ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ ++ char *bufpos; /* Next read position of BUF. */ ++ size_t buflen; /* The length of the byte sequence in buf. */ ++ wint_t wc = 0; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character which shows ++ as same character as WC. */ ++ mbstate_t state; /* State of the stream. */ ++ bool convfail = false; /* true, when conversion failed. Otherwise false. */ ++ ++ current_rp = frp; ++ ++ found_any_selected_field = 0; ++ field_idx = 1; ++ bufpos = buf; ++ buflen = 0; ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ c = getc (stream); ++ empty_input = (c == EOF); ++ if (c != EOF) ++ { ++ ungetc (c, stream); ++ wc = 0; ++ } ++ else ++ wc = WEOF; ++ ++ /* To support the semantics of the -s flag, we may have to buffer ++ all of the first field to determine whether it is `delimited.' ++ But that is unnecessary if all non-delimited lines must be printed ++ and the first field has been selected, or if non-delimited lines ++ must be suppressed and the first field has *not* been selected. ++ That is because a non-delimited line has exactly one field. */ ++ buffer_first_field = (suppress_non_delimited ^ !print_kth (1)); ++ ++ while (1) ++ { ++ if (field_idx == 1 && buffer_first_field) ++ { ++ int len = 0; ++ ++ while (1) ++ { ++ REFILL_BUFFER (buf, bufpos, buflen, stream); ++ ++ GET_NEXT_WC_FROM_BUFFER ++ (wc, bufpos, buflen, mblength, state, convfail); ++ ++ if (wc == WEOF) ++ break; ++ ++ field_1_buffer = xrealloc (field_1_buffer, len + mblength); ++ memcpy (field_1_buffer + len, bufpos, mblength); ++ len += mblength; ++ buflen -= mblength; ++ bufpos += mblength; ++ ++ if (!convfail && (wc == line_delim || wc == wcdelim)) ++ break; ++ } ++ ++ if (len <= 0 && wc == WEOF) ++ break; ++ ++ /* If the first field extends to the end of line (it is not ++ delimited) and we are printing all non-delimited lines, ++ print this one. */ ++ if (convfail || (!convfail && wc != wcdelim)) ++ { ++ if (suppress_non_delimited) ++ { ++ /* Empty. */ ++ } ++ else ++ { ++ fwrite (field_1_buffer, sizeof (char), len, stdout); ++ /* Make sure the output line is newline terminated. */ ++ if (convfail || (!convfail && wc != line_delim)) ++ putchar (line_delim); ++ } ++ continue; ++ } ++ ++ if (print_kth (1)) ++ { ++ /* Print the field, but not the trailing delimiter. */ ++ fwrite (field_1_buffer, sizeof (char), len - 1, stdout); ++ found_any_selected_field = 1; ++ } ++ next_item (&field_idx); ++ } ++ ++ if (wc != WEOF) ++ { ++ if (print_kth (field_idx)) ++ { ++ if (found_any_selected_field) ++ { ++ fwrite (output_delimiter_string, sizeof (char), ++ output_delimiter_length, stdout); ++ } ++ found_any_selected_field = 1; ++ } ++ ++ while (1) ++ { ++ REFILL_BUFFER (buf, bufpos, buflen, stream); ++ ++ GET_NEXT_WC_FROM_BUFFER ++ (wc, bufpos, buflen, mblength, state, convfail); ++ ++ if (wc == WEOF) ++ break; ++ else if (!convfail && (wc == wcdelim || wc == line_delim)) ++ { ++ buflen -= mblength; ++ bufpos += mblength; ++ break; ++ } ++ ++ if (print_kth (field_idx)) ++ fwrite (bufpos, mblength, sizeof(char), stdout); ++ ++ buflen -= mblength; ++ bufpos += mblength; ++ } ++ } ++ ++ if ((!convfail || wc == line_delim) && buflen < 1) ++ wc = WEOF; ++ ++ if (!convfail && wc == wcdelim) ++ next_item (&field_idx); ++ else if (wc == WEOF || (!convfail && wc == line_delim)) ++ { ++ if (found_any_selected_field ++ || (!empty_input && !(suppress_non_delimited && field_idx == 1))) ++ putchar (line_delim); ++ if (wc == WEOF) ++ break; ++ field_idx = 1; ++ current_rp = frp; ++ found_any_selected_field = 0; ++ } ++ } ++} ++#endif ++ + static void + cut_stream (FILE *stream) + { +- if (operating_mode == byte_mode) +- cut_bytes (stream); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1 && !force_singlebyte_mode) ++ { ++ switch (operating_mode) ++ { ++ case byte_mode: ++ if (byte_mode_character_aware) ++ cut_characters_or_cut_bytes_no_split (stream); ++ else ++ cut_bytes (stream); ++ break; ++ ++ case character_mode: ++ cut_characters_or_cut_bytes_no_split (stream); ++ break; ++ ++ case field_mode: ++ if (delimlen == 1) ++ { ++ /* Check if we have utf8 multibyte locale, so we can use this ++ optimization because of uniqueness of characters, which is ++ not true for e.g. SJIS */ ++ char * loc = setlocale(LC_CTYPE, NULL); ++ if (loc && (strstr (loc, "UTF-8") || strstr (loc, "utf-8") || ++ strstr (loc, "UTF8") || strstr (loc, "utf8"))) ++ { ++ cut_fields (stream); ++ break; ++ } ++ } ++ cut_fields_mb (stream); ++ break; ++ ++ default: ++ abort (); ++ } ++ } + else +- cut_fields (stream); ++#endif ++ { ++ if (operating_mode == field_mode) ++ cut_fields (stream); ++ else ++ cut_bytes (stream); ++ } + } + + /* Process file FILE to standard output. +@@ -483,6 +836,7 @@ main (int argc, char **argv) + bool ok; + bool delim_specified = false; + char *spec_list_string IF_LINT ( = NULL); ++ char mbdelim[MB_LEN_MAX + 1]; + + initialize_main (&argc, &argv); + set_program_name (argv[0]); +@@ -505,7 +859,6 @@ main (int argc, char **argv) + switch (optc) + { + case 'b': +- case 'c': + /* Build the byte list. */ + if (operating_mode != undefined_mode) + FATAL_ERROR (_("only one type of list may be specified")); +@@ -513,6 +866,14 @@ main (int argc, char **argv) + spec_list_string = optarg; + break; + ++ case 'c': ++ /* Build the character list. */ ++ if (operating_mode != undefined_mode) ++ FATAL_ERROR (_("only one type of list may be specified")); ++ operating_mode = character_mode; ++ spec_list_string = optarg; ++ break; ++ + case 'f': + /* Build the field list. */ + if (operating_mode != undefined_mode) +@@ -524,10 +885,38 @@ main (int argc, char **argv) + case 'd': + /* New delimiter. */ + /* Interpret -d '' to mean 'use the NUL byte as the delimiter.' */ +- if (optarg[0] != '\0' && optarg[1] != '\0') +- FATAL_ERROR (_("the delimiter must be a single character")); +- delim = optarg[0]; +- delim_specified = true; ++ { ++#if HAVE_MBRTOWC ++ if(MB_CUR_MAX > 1) ++ { ++ mbstate_t state; ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ delimlen = mbrtowc (&wcdelim, optarg, strnlen(optarg, MB_LEN_MAX), &state); ++ ++ if (delimlen == (size_t)-1 || delimlen == (size_t)-2) ++ ++force_singlebyte_mode; ++ else ++ { ++ delimlen = (delimlen < 1) ? 1 : delimlen; ++ if (wcdelim != L'\0' && *(optarg + delimlen) != '\0') ++ FATAL_ERROR (_("the delimiter must be a single character")); ++ memcpy (mbdelim, optarg, delimlen); ++ mbdelim[delimlen] = '\0'; ++ if (delimlen == 1) ++ delim = *optarg; ++ } ++ } ++ ++ if (MB_CUR_MAX <= 1 || force_singlebyte_mode) ++#endif ++ { ++ if (optarg[0] != '\0' && optarg[1] != '\0') ++ FATAL_ERROR (_("the delimiter must be a single character")); ++ delim = (unsigned char) optarg[0]; ++ } ++ delim_specified = true; ++ } + break; + + case OUTPUT_DELIMITER_OPTION: +@@ -540,6 +929,7 @@ main (int argc, char **argv) + break; + + case 'n': ++ byte_mode_character_aware = 1; + break; + + case 's': +@@ -579,15 +969,34 @@ main (int argc, char **argv) + | (complement ? SETFLD_COMPLEMENT : 0) ); + + if (!delim_specified) +- delim = '\t'; ++ { ++ delim = '\t'; ++#ifdef HAVE_MBRTOWC ++ wcdelim = L'\t'; ++ mbdelim[0] = '\t'; ++ mbdelim[1] = '\0'; ++ delimlen = 1; ++#endif ++ } + + if (output_delimiter_string == NULL) + { +- static char dummy[2]; +- dummy[0] = delim; +- dummy[1] = '\0'; +- output_delimiter_string = dummy; +- output_delimiter_length = 1; ++#ifdef HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1 && !force_singlebyte_mode) ++ { ++ output_delimiter_string = xstrdup(mbdelim); ++ output_delimiter_length = delimlen; ++ } ++ ++ if (MB_CUR_MAX <= 1 || force_singlebyte_mode) ++#endif ++ { ++ static char dummy[2]; ++ dummy[0] = delim; ++ dummy[1] = '\0'; ++ output_delimiter_string = dummy; ++ output_delimiter_length = 1; ++ } + } + + if (optind == argc) +diff -Naurp coreutils-8.27-orig/src/expand.c coreutils-8.27/src/expand.c +--- coreutils-8.27-orig/src/expand.c 2017-02-26 15:42:25.000000000 -0600 ++++ coreutils-8.27/src/expand.c 2017-03-11 23:49:06.758133530 -0600 +@@ -37,6 +37,9 @@ + #include + #include + #include ++ ++#include ++ + #include "system.h" + #include "die.h" + #include "xstrndup.h" +@@ -100,19 +103,41 @@ expand (void) + { + /* Input stream. */ + FILE *fp = next_file (NULL); ++ mb_file_t mbf; ++ mbf_char_t c; ++ /* True if the starting locale is utf8. */ ++ bool using_utf_locale; ++ ++ /* True if the first file contains BOM header. */ ++ bool found_bom; ++ using_utf_locale=check_utf_locale(); + + if (!fp) + return; ++ mbf_init (mbf, fp); ++ found_bom=check_bom(fp,&mbf); + +- while (true) ++ if (using_utf_locale == false && found_bom == true) ++ { ++ /*try using some predefined locale */ ++ ++ if (set_utf_locale () != 0) + { +- /* Input character, or EOF. */ +- int c; ++ error (EXIT_FAILURE, errno, _("cannot set UTF-8 locale")); ++ } ++ } ++ + ++ if (found_bom == true) ++ { ++ print_bom(); ++ } ++ ++ while (true) ++ { + /* If true, perform translations. */ + bool convert = true; + +- + /* The following variables have valid values only when CONVERT + is true: */ + +@@ -122,17 +147,48 @@ expand (void) + /* Index in TAB_LIST of next tab stop to examine. */ + size_t tab_index = 0; + +- + /* Convert a line of text. */ + + do + { +- while ((c = getc (fp)) < 0 && (fp = next_file (fp))) +- continue; ++ while (true) { ++ mbf_getc (c, mbf); ++ if ((mb_iseof (c)) && (fp = next_file (fp))) ++ { ++ mbf_init (mbf, fp); ++ if (fp!=NULL) ++ { ++ if (check_bom(fp,&mbf)==true) ++ { ++ /*Not the first file - check BOM header*/ ++ if (using_utf_locale==false && found_bom==false) ++ { ++ /*BOM header in subsequent file but not in the first one. */ ++ error (EXIT_FAILURE, errno, _("combination of files with and without BOM header")); ++ } ++ } ++ else ++ { ++ if(using_utf_locale==false && found_bom==true) ++ { ++ /*First file conatined BOM header - locale was switched to UTF ++ /*all subsequent files should contain BOM. */ ++ error (EXIT_FAILURE, errno, _("combination of files with and without BOM header")); ++ } ++ } ++ } ++ continue; ++ } ++ else ++ { ++ break; ++ } ++ } ++ + + if (convert) + { +- if (c == '\t') ++ if (mb_iseq (c, '\t')) + { + /* Column the next input tab stop is on. */ + uintmax_t next_tab_column; +@@ -151,32 +207,34 @@ expand (void) + if (putchar (' ') < 0) + die (EXIT_FAILURE, errno, _("write error")); + +- c = ' '; ++ mb_setascii (&c, ' '); + } +- else if (c == '\b') ++ else if (mb_iseq (c, '\b')) + { + /* Go back one column, and force recalculation of the + next tab stop. */ + column -= !!column; + tab_index -= !!tab_index; + } +- else ++ /* A leading control character could make us trip over. */ ++ else if (!mb_iscntrl (c)) + { +- column++; ++ column += mb_width (c); + if (!column) + die (EXIT_FAILURE, 0, _("input line is too long")); + } + +- convert &= convert_entire_line || !! isblank (c); ++ convert &= convert_entire_line || mb_isblank (c); + } + +- if (c < 0) ++ if (mb_iseof (c)) + return; + +- if (putchar (c) < 0) ++ mb_putc (c, stdout); ++ if (ferror (stdout)) + die (EXIT_FAILURE, errno, _("write error")); + } +- while (c != '\n'); ++ while (!mb_iseq (c, '\n')); + } + } + +diff -Naurp coreutils-8.27-orig/src/expand-common.c coreutils-8.27/src/expand-common.c +--- coreutils-8.27-orig/src/expand-common.c 2017-03-01 11:22:55.000000000 -0600 ++++ coreutils-8.27/src/expand-common.c 2017-03-11 23:49:06.757133570 -0600 +@@ -18,6 +18,7 @@ + + #include + #include ++#include + #include "system.h" + #include "die.h" + #include "error.h" +@@ -105,6 +106,119 @@ set_extend_size (uintmax_t tabval) + return ok; + } + ++extern int ++set_utf_locale (void) ++{ ++ /*try using some predefined locale */ ++ const char* predef_locales[] = {"C.UTF8","en_US.UTF8","en_GB.UTF8"}; ++ ++ const int predef_locales_count=3; ++ for (int i=0;ibufcount=0; ++ if (c == 0xEF) ++ { ++ c=fgetc(fp); ++ } ++ else ++ { ++ if (c != EOF) ++ { ++ ungetc(c,fp); ++ } ++ return false; ++ } ++ ++ if (c == 0xBB) ++ { ++ c=fgetc(fp); ++ } ++ else ++ { ++ if ( c!= EOF ) ++ { ++ mbf->buf[0]=(unsigned char) 0xEF; ++ mbf->bufcount=1; ++ ungetc(c,fp); ++ return false; ++ } ++ else ++ { ++ ungetc(0xEF,fp); ++ return false; ++ } ++ } ++ if (c == 0xBF) ++ { ++ mbf->bufcount=0; ++ return true; ++ } ++ else ++ { ++ if (c != EOF) ++ { ++ mbf->buf[0]=(unsigned char) 0xEF; ++ mbf->buf[1]=(unsigned char) 0xBB; ++ mbf->bufcount=2; ++ ungetc(c,fp); ++ return false; ++ } ++ else ++ { ++ mbf->buf[0]=(unsigned char) 0xEF; ++ mbf->bufcount=1; ++ ungetc(0xBB,fp); ++ return false; ++ } ++ } ++ return false; ++} ++ ++extern void ++print_bom(void) ++{ ++ putc (0xEF, stdout); ++ putc (0xBB, stdout); ++ putc (0xBF, stdout); ++} ++ + /* Add the comma or blank separated list of tab stops STOPS + to the list of tab stops. */ + extern void +diff -Naurp coreutils-8.27-orig/src/expand-common.h coreutils-8.27/src/expand-common.h +--- coreutils-8.27-orig/src/expand-common.h 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/expand-common.h 2017-03-11 23:49:06.758133530 -0600 +@@ -34,6 +34,18 @@ extern size_t max_column_width; + /* The desired exit status. */ + extern int exit_status; + ++extern int ++set_utf_locale (void); ++ ++extern bool ++check_utf_locale(void); ++ ++extern bool ++check_bom(FILE* fp, mb_file_t *mbf); ++ ++extern void ++print_bom(void); ++ + /* Add tab stop TABVAL to the end of 'tab_list'. */ + extern void + add_tab_stop (uintmax_t tabval); +diff -Naurp coreutils-8.27-orig/src/fold.c coreutils-8.27/src/fold.c +--- coreutils-8.27-orig/src/fold.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/fold.c 2017-03-11 23:49:30.982169404 -0600 +@@ -22,12 +22,34 @@ + #include + #include + ++/* Get mbstate_t, mbrtowc(), wcwidth(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ ++/* Get iswprint(), iswblank(), wcwidth(). */ ++#if HAVE_WCTYPE_H ++# include ++#endif ++ + #include "system.h" + #include "die.h" + #include "error.h" + #include "fadvise.h" + #include "xdectoint.h" + ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 ++# undef MB_LEN_MAX ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ + #define TAB_WIDTH 8 + + /* The official name of this program (e.g., no 'g' prefix). */ +@@ -35,20 +57,41 @@ + + #define AUTHORS proper_name ("David MacKenzie") + ++#define FATAL_ERROR(Message) \ ++ do \ ++ { \ ++ error (0, 0, (Message)); \ ++ usage (2); \ ++ } \ ++ while (0) ++ ++enum operating_mode ++{ ++ /* Fold texts by columns that are at the given positions. */ ++ column_mode, ++ ++ /* Fold texts by bytes that are at the given positions. */ ++ byte_mode, ++ ++ /* Fold texts by characters that are at the given positions. */ ++ character_mode, ++}; ++ ++/* The argument shows current mode. (Default: column_mode) */ ++static enum operating_mode operating_mode; ++ + /* If nonzero, try to break on whitespace. */ + static bool break_spaces; + +-/* If nonzero, count bytes, not column positions. */ +-static bool count_bytes; +- + /* If nonzero, at least one of the files we read was standard input. */ + static bool have_read_stdin; + +-static char const shortopts[] = "bsw:0::1::2::3::4::5::6::7::8::9::"; ++static char const shortopts[] = "bcsw:0::1::2::3::4::5::6::7::8::9::"; + + static struct option const longopts[] = + { + {"bytes", no_argument, NULL, 'b'}, ++ {"characters", no_argument, NULL, 'c'}, + {"spaces", no_argument, NULL, 's'}, + {"width", required_argument, NULL, 'w'}, + {GETOPT_HELP_OPTION_DECL}, +@@ -76,6 +119,7 @@ Wrap input lines in each FILE, writing t + + fputs (_("\ + -b, --bytes count bytes rather than columns\n\ ++ -c, --characters count characters rather than columns\n\ + -s, --spaces break at spaces\n\ + -w, --width=WIDTH use WIDTH columns instead of 80\n\ + "), stdout); +@@ -93,7 +137,7 @@ Wrap input lines in each FILE, writing t + static size_t + adjust_column (size_t column, char c) + { +- if (!count_bytes) ++ if (operating_mode != byte_mode) + { + if (c == '\b') + { +@@ -116,30 +160,14 @@ adjust_column (size_t column, char c) + to stdout, with maximum line length WIDTH. + Return true if successful. */ + +-static bool +-fold_file (char const *filename, size_t width) ++static void ++fold_text (FILE *istream, size_t width, int *saved_errno) + { +- FILE *istream; + int c; + size_t column = 0; /* Screen column where next char will go. */ + size_t offset_out = 0; /* Index in 'line_out' for next char. */ + static char *line_out = NULL; + static size_t allocated_out = 0; +- int saved_errno; +- +- if (STREQ (filename, "-")) +- { +- istream = stdin; +- have_read_stdin = true; +- } +- else +- istream = fopen (filename, "r"); +- +- if (istream == NULL) +- { +- error (0, errno, "%s", quotef (filename)); +- return false; +- } + + fadvise (istream, FADVISE_SEQUENTIAL); + +@@ -169,6 +197,15 @@ fold_file (char const *filename, size_t + bool found_blank = false; + size_t logical_end = offset_out; + ++ /* If LINE_OUT has no wide character, ++ put a new wide character in LINE_OUT ++ if column is bigger than width. */ ++ if (offset_out == 0) ++ { ++ line_out[offset_out++] = c; ++ continue; ++ } ++ + /* Look for the last blank. */ + while (logical_end) + { +@@ -215,11 +252,220 @@ fold_file (char const *filename, size_t + line_out[offset_out++] = c; + } + +- saved_errno = errno; ++ *saved_errno = errno; ++ ++ if (offset_out) ++ fwrite (line_out, sizeof (char), (size_t) offset_out, stdout); ++ ++} ++ ++#if HAVE_MBRTOWC ++static void ++fold_multibyte_text (FILE *istream, size_t width, int *saved_errno) ++{ ++ char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */ ++ size_t buflen = 0; /* The length of the byte sequence in buf. */ ++ char *bufpos = buf; /* Next read position of BUF. */ ++ wint_t wc; /* A gotten wide character. */ ++ size_t mblength; /* The byte size of a multibyte character which shows ++ as same character as WC. */ ++ mbstate_t state, state_bak; /* State of the stream. */ ++ int convfail = 0; /* 1, when conversion is failed. Otherwise 0. */ ++ ++ static char *line_out = NULL; ++ size_t offset_out = 0; /* Index in `line_out' for next char. */ ++ static size_t allocated_out = 0; ++ ++ int increment; ++ size_t column = 0; ++ ++ size_t last_blank_pos; ++ size_t last_blank_column; ++ int is_blank_seen; ++ int last_blank_increment = 0; ++ int is_bs_following_last_blank; ++ size_t bs_following_last_blank_num; ++ int is_cr_after_last_blank; ++ ++#define CLEAR_FLAGS \ ++ do \ ++ { \ ++ last_blank_pos = 0; \ ++ last_blank_column = 0; \ ++ is_blank_seen = 0; \ ++ is_bs_following_last_blank = 0; \ ++ bs_following_last_blank_num = 0; \ ++ is_cr_after_last_blank = 0; \ ++ } \ ++ while (0) ++ ++#define START_NEW_LINE \ ++ do \ ++ { \ ++ putchar ('\n'); \ ++ column = 0; \ ++ offset_out = 0; \ ++ CLEAR_FLAGS; \ ++ } \ ++ while (0) ++ ++ CLEAR_FLAGS; ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ for (;; bufpos += mblength, buflen -= mblength) ++ { ++ if (buflen < MB_LEN_MAX && !feof (istream) && !ferror (istream)) ++ { ++ memmove (buf, bufpos, buflen); ++ buflen += fread (buf + buflen, sizeof(char), BUFSIZ, istream); ++ bufpos = buf; ++ } ++ ++ if (buflen < 1) ++ break; ++ ++ /* Get a wide character. */ ++ state_bak = state; ++ mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &state); ++ ++ switch (mblength) ++ { ++ case (size_t)-1: ++ case (size_t)-2: ++ convfail++; ++ state = state_bak; ++ /* Fall through. */ ++ ++ case 0: ++ mblength = 1; ++ break; ++ } ++ ++rescan: ++ if (convfail) ++ increment = 1; ++ else if (wc == L'\n') ++ { ++ /* preserve newline */ ++ fwrite (line_out, sizeof(char), offset_out, stdout); ++ START_NEW_LINE; ++ continue; ++ } ++ else if (operating_mode == byte_mode) /* byte mode */ ++ increment = mblength; ++ else if (operating_mode == character_mode) /* character mode */ ++ increment = 1; ++ else /* column mode */ ++ { ++ switch (wc) ++ { ++ case L'\b': ++ increment = (column > 0) ? -1 : 0; ++ break; ++ ++ case L'\r': ++ increment = -1 * column; ++ break; ++ ++ case L'\t': ++ increment = 8 - column % 8; ++ break; ++ ++ default: ++ increment = wcwidth (wc); ++ increment = (increment < 0) ? 0 : increment; ++ } ++ } ++ ++ if (column + increment > width && break_spaces && last_blank_pos) ++ { ++ fwrite (line_out, sizeof(char), last_blank_pos, stdout); ++ putchar ('\n'); ++ ++ offset_out = offset_out - last_blank_pos; ++ column = column - last_blank_column + ((is_cr_after_last_blank) ++ ? last_blank_increment : bs_following_last_blank_num); ++ memmove (line_out, line_out + last_blank_pos, offset_out); ++ CLEAR_FLAGS; ++ goto rescan; ++ } ++ ++ if (column + increment > width && column != 0) ++ { ++ fwrite (line_out, sizeof(char), offset_out, stdout); ++ START_NEW_LINE; ++ goto rescan; ++ } ++ ++ if (allocated_out < offset_out + mblength) ++ { ++ line_out = X2REALLOC (line_out, &allocated_out); ++ } ++ ++ memcpy (line_out + offset_out, bufpos, mblength); ++ offset_out += mblength; ++ column += increment; ++ ++ if (is_blank_seen && !convfail && wc == L'\r') ++ is_cr_after_last_blank = 1; ++ ++ if (is_bs_following_last_blank && !convfail && wc == L'\b') ++ ++bs_following_last_blank_num; ++ else ++ is_bs_following_last_blank = 0; ++ ++ if (break_spaces && !convfail && iswblank (wc)) ++ { ++ last_blank_pos = offset_out; ++ last_blank_column = column; ++ is_blank_seen = 1; ++ last_blank_increment = increment; ++ is_bs_following_last_blank = 1; ++ bs_following_last_blank_num = 0; ++ is_cr_after_last_blank = 0; ++ } ++ } ++ ++ *saved_errno = errno; + + if (offset_out) + fwrite (line_out, sizeof (char), (size_t) offset_out, stdout); + ++} ++#endif ++ ++/* Fold file FILENAME, or standard input if FILENAME is "-", ++ to stdout, with maximum line length WIDTH. ++ Return 0 if successful, 1 if an error occurs. */ ++ ++static bool ++fold_file (char const *filename, size_t width) ++{ ++ FILE *istream; ++ int saved_errno; ++ ++ if (STREQ (filename, "-")) ++ { ++ istream = stdin; ++ have_read_stdin = 1; ++ } ++ else ++ istream = fopen (filename, "r"); ++ ++ if (istream == NULL) ++ { ++ error (0, errno, "%s", filename); ++ return 1; ++ } ++ ++ /* Define how ISTREAM is being folded. */ ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ fold_multibyte_text (istream, width, &saved_errno); ++ else ++#endif ++ fold_text (istream, width, &saved_errno); ++ + if (ferror (istream)) + { + error (0, saved_errno, "%s", quotef (filename)); +@@ -252,7 +498,8 @@ main (int argc, char **argv) + + atexit (close_stdout); + +- break_spaces = count_bytes = have_read_stdin = false; ++ operating_mode = column_mode; ++ break_spaces = have_read_stdin = false; + + while ((optc = getopt_long (argc, argv, shortopts, longopts, NULL)) != -1) + { +@@ -261,7 +508,15 @@ main (int argc, char **argv) + switch (optc) + { + case 'b': /* Count bytes rather than columns. */ +- count_bytes = true; ++ if (operating_mode != column_mode) ++ FATAL_ERROR (_("only one way of folding may be specified")); ++ operating_mode = byte_mode; ++ break; ++ ++ case 'c': ++ if (operating_mode != column_mode) ++ FATAL_ERROR (_("only one way of folding may be specified")); ++ operating_mode = character_mode; + break; + + case 's': /* Break at word boundaries. */ +diff -Naurp coreutils-8.27-orig/src/join.c coreutils-8.27/src/join.c +--- coreutils-8.27-orig/src/join.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/join.c 2017-03-11 23:47:13.091286290 -0600 +@@ -22,19 +22,33 @@ + #include + #include + ++/* Get mbstate_t, mbrtowc(), mbrtowc(), wcwidth(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ ++/* Get iswblank(), towupper. */ ++#if HAVE_WCTYPE_H ++# include ++#endif ++ + #include "system.h" + #include "die.h" + #include "error.h" + #include "fadvise.h" + #include "hard-locale.h" + #include "linebuffer.h" +-#include "memcasecmp.h" + #include "quote.h" + #include "stdio--.h" + #include "xmemcoll.h" + #include "xstrtol.h" + #include "argmatch.h" + ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ + /* The official name of this program (e.g., no 'g' prefix). */ + #define PROGRAM_NAME "join" + +@@ -136,10 +150,12 @@ static struct outlist outlist_head; + /* Last element in 'outlist', where a new element can be added. */ + static struct outlist *outlist_end = &outlist_head; + +-/* Tab character separating fields. If negative, fields are separated +- by any nonempty string of blanks, otherwise by exactly one +- tab character whose value (when cast to unsigned char) equals TAB. */ +-static int tab = -1; ++/* Tab character separating fields. If NULL, fields are separated ++ by any nonempty string of blanks. */ ++static char *tab = NULL; ++ ++/* The number of bytes used for tab. */ ++static size_t tablen = 0; + + /* If nonzero, check that the input is correctly ordered. */ + static enum +@@ -276,13 +292,14 @@ xfields (struct line *line) + if (ptr == lim) + return; + +- if (0 <= tab && tab != '\n') ++ if (tab != NULL) + { ++ unsigned char t = tab[0]; + char *sep; +- for (; (sep = memchr (ptr, tab, lim - ptr)) != NULL; ptr = sep + 1) ++ for (; (sep = memchr (ptr, t, lim - ptr)) != NULL; ptr = sep + 1) + extract_field (line, ptr, sep - ptr); + } +- else if (tab < 0) ++ else + { + /* Skip leading blanks before the first field. */ + while (field_sep (*ptr)) +@@ -306,6 +323,147 @@ xfields (struct line *line) + extract_field (line, ptr, lim - ptr); + } + ++#if HAVE_MBRTOWC ++static void ++xfields_multibyte (struct line *line) ++{ ++ char *ptr = line->buf.buffer; ++ char const *lim = ptr + line->buf.length - 1; ++ wchar_t wc = 0; ++ size_t mblength = 1; ++ mbstate_t state, state_bak; ++ ++ memset (&state, 0, sizeof (mbstate_t)); ++ ++ if (ptr >= lim) ++ return; ++ ++ if (tab != NULL) ++ { ++ char *sep = ptr; ++ for (; ptr < lim; ptr = sep + mblength) ++ { ++ sep = ptr; ++ while (sep < lim) ++ { ++ state_bak = state; ++ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state); ++ ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ if (mblength == tablen && !memcmp (sep, tab, mblength)) ++ break; ++ else ++ { ++ sep += mblength; ++ continue; ++ } ++ } ++ ++ if (sep >= lim) ++ break; ++ ++ extract_field (line, ptr, sep - ptr); ++ } ++ } ++ else ++ { ++ /* Skip leading blanks before the first field. */ ++ while(ptr < lim) ++ { ++ state_bak = state; ++ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state); ++ ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ break; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ if (!iswblank(wc) && wc != '\n') ++ break; ++ ptr += mblength; ++ } ++ ++ do ++ { ++ char *sep; ++ state_bak = state; ++ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state); ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ break; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ sep = ptr + mblength; ++ while (sep < lim) ++ { ++ state_bak = state; ++ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state); ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ break; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ if (iswblank (wc) || wc == '\n') ++ break; ++ ++ sep += mblength; ++ } ++ ++ extract_field (line, ptr, sep - ptr); ++ if (sep >= lim) ++ return; ++ ++ state_bak = state; ++ mblength = mbrtowc (&wc, sep, lim - sep + 1, &state); ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ break; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ ptr = sep + mblength; ++ while (ptr < lim) ++ { ++ state_bak = state; ++ mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state); ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ mblength = 1; ++ state = state_bak; ++ break; ++ } ++ mblength = (mblength < 1) ? 1 : mblength; ++ ++ if (!iswblank (wc) && wc != '\n') ++ break; ++ ++ ptr += mblength; ++ } ++ } ++ while (ptr < lim); ++ } ++ ++ extract_field (line, ptr, lim - ptr); ++} ++#endif ++ + static void + freeline (struct line *line) + { +@@ -327,56 +485,133 @@ keycmp (struct line const *line1, struct + size_t jf_1, size_t jf_2) + { + /* Start of field to compare in each file. */ +- char *beg1; +- char *beg2; +- +- size_t len1; +- size_t len2; /* Length of fields to compare. */ ++ char *beg[2]; ++ char *copy[2]; ++ size_t len[2]; /* Length of fields to compare. */ + int diff; ++ int i, j; ++ int mallocd = 0; + + if (jf_1 < line1->nfields) + { +- beg1 = line1->fields[jf_1].beg; +- len1 = line1->fields[jf_1].len; ++ beg[0] = line1->fields[jf_1].beg; ++ len[0] = line1->fields[jf_1].len; + } + else + { +- beg1 = NULL; +- len1 = 0; ++ beg[0] = NULL; ++ len[0] = 0; + } + + if (jf_2 < line2->nfields) + { +- beg2 = line2->fields[jf_2].beg; +- len2 = line2->fields[jf_2].len; ++ beg[1] = line2->fields[jf_2].beg; ++ len[1] = line2->fields[jf_2].len; + } + else + { +- beg2 = NULL; +- len2 = 0; ++ beg[1] = NULL; ++ len[1] = 0; + } + +- if (len1 == 0) +- return len2 == 0 ? 0 : -1; +- if (len2 == 0) ++ if (len[0] == 0) ++ return len[1] == 0 ? 0 : -1; ++ if (len[1] == 0) + return 1; + + if (ignore_case) + { +- /* FIXME: ignore_case does not work with NLS (in particular, +- with multibyte chars). */ +- diff = memcasecmp (beg1, beg2, MIN (len1, len2)); ++#ifdef HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ size_t mblength; ++ wchar_t wc, uwc; ++ mbstate_t state, state_bak; ++ ++ memset (&state, '\0', sizeof (mbstate_t)); ++ ++ for (i = 0; i < 2; i++) ++ { ++ mallocd = 1; ++ copy[i] = xmalloc (len[i] + 1); ++ memset (copy[i], '\0',len[i] + 1); ++ ++ for (j = 0; j < MIN (len[0], len[1]);) ++ { ++ state_bak = state; ++ mblength = mbrtowc (&wc, beg[i] + j, len[i] - j, &state); ++ ++ switch (mblength) ++ { ++ case (size_t) -1: ++ case (size_t) -2: ++ state = state_bak; ++ /* Fall through */ ++ case 0: ++ mblength = 1; ++ break; ++ ++ default: ++ uwc = towupper (wc); ++ ++ if (uwc != wc) ++ { ++ mbstate_t state_wc; ++ size_t mblen; ++ ++ memset (&state_wc, '\0', sizeof (mbstate_t)); ++ mblen = wcrtomb (copy[i] + j, uwc, &state_wc); ++ assert (mblen != (size_t)-1); ++ } ++ else ++ memcpy (copy[i] + j, beg[i] + j, mblength); ++ } ++ j += mblength; ++ } ++ copy[i][j] = '\0'; ++ } ++ } ++ else ++#endif ++ { ++ for (i = 0; i < 2; i++) ++ { ++ mallocd = 1; ++ copy[i] = xmalloc (len[i] + 1); ++ ++ for (j = 0; j < MIN (len[0], len[1]); j++) ++ copy[i][j] = toupper (beg[i][j]); ++ ++ copy[i][j] = '\0'; ++ } ++ } + } + else + { +- if (hard_LC_COLLATE) +- return xmemcoll (beg1, len1, beg2, len2); +- diff = memcmp (beg1, beg2, MIN (len1, len2)); ++ copy[0] = beg[0]; ++ copy[1] = beg[1]; + } + ++ if (hard_LC_COLLATE) ++ { ++ diff = xmemcoll ((char *) copy[0], len[0], (char *) copy[1], len[1]); ++ ++ if (mallocd) ++ for (i = 0; i < 2; i++) ++ free (copy[i]); ++ ++ return diff; ++ } ++ diff = memcmp (copy[0], copy[1], MIN (len[0], len[1])); ++ ++ if (mallocd) ++ for (i = 0; i < 2; i++) ++ free (copy[i]); ++ ++ + if (diff) + return diff; +- return len1 < len2 ? -1 : len1 != len2; ++ return len[0] - len[1]; + } + + /* Check that successive input lines PREV and CURRENT from input file +@@ -468,6 +703,11 @@ get_line (FILE *fp, struct line **linep, + } + ++line_no[which - 1]; + ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ xfields_multibyte (line); ++ else ++#endif + xfields (line); + + if (prevline[which - 1]) +@@ -567,21 +807,28 @@ prfield (size_t n, struct line const *li + + /* Output all the fields in line, other than the join field. */ + ++#define PUT_TAB_CHAR \ ++ do \ ++ { \ ++ (tab != NULL) ? \ ++ fwrite(tab, sizeof(char), tablen, stdout) : putchar (' '); \ ++ } \ ++ while (0) ++ + static void + prfields (struct line const *line, size_t join_field, size_t autocount) + { + size_t i; + size_t nfields = autoformat ? autocount : line->nfields; +- char output_separator = tab < 0 ? ' ' : tab; + + for (i = 0; i < join_field && i < nfields; ++i) + { +- putchar (output_separator); ++ PUT_TAB_CHAR; + prfield (i, line); + } + for (i = join_field + 1; i < nfields; ++i) + { +- putchar (output_separator); ++ PUT_TAB_CHAR; + prfield (i, line); + } + } +@@ -592,7 +839,6 @@ static void + prjoin (struct line const *line1, struct line const *line2) + { + const struct outlist *outlist; +- char output_separator = tab < 0 ? ' ' : tab; + size_t field; + struct line const *line; + +@@ -626,7 +872,7 @@ prjoin (struct line const *line1, struct + o = o->next; + if (o == NULL) + break; +- putchar (output_separator); ++ PUT_TAB_CHAR; + } + putchar (eolchar); + } +@@ -1104,20 +1350,43 @@ main (int argc, char **argv) + + case 't': + { +- unsigned char newtab = optarg[0]; ++ char *newtab = NULL; ++ size_t newtablen; ++ newtab = xstrdup (optarg); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ mbstate_t state; ++ ++ memset (&state, 0, sizeof (mbstate_t)); ++ newtablen = mbrtowc (NULL, newtab, ++ strnlen (newtab, MB_LEN_MAX), ++ &state); ++ if (newtablen == (size_t) 0 ++ || newtablen == (size_t) -1 ++ || newtablen == (size_t) -2) ++ newtablen = 1; ++ } ++ else ++#endif ++ newtablen = 1; + if (! newtab) +- newtab = '\n'; /* '' => process the whole line. */ ++ newtab = (char*)"\n"; /* '' => process the whole line. */ + else if (optarg[1]) + { +- if (STREQ (optarg, "\\0")) +- newtab = '\0'; +- else +- die (EXIT_FAILURE, 0, _("multi-character tab %s"), +- quote (optarg)); ++ if (newtablen == 1 && newtab[1]) ++ { ++ if (STREQ (newtab, "\\0")) ++ newtab[0] = '\0'; ++ } ++ } ++ if (tab != NULL && strcmp (tab, newtab)) ++ { ++ free (newtab); ++ die (EXIT_FAILURE, 0, _("incompatible tabs")); + } +- if (0 <= tab && tab != newtab) +- die (EXIT_FAILURE, 0, _("incompatible tabs")); + tab = newtab; ++ tablen = newtablen; + } + break; + +diff -Naurp coreutils-8.27-orig/src/pr.c coreutils-8.27/src/pr.c +--- coreutils-8.27-orig/src/pr.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/pr.c 2017-03-11 23:47:13.094286139 -0600 +@@ -311,6 +311,24 @@ + + #include + #include ++ ++/* Get MB_LEN_MAX. */ ++#include ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX == 1 ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Get MB_CUR_MAX. */ ++#include ++ ++/* Solaris 2.5 has a bug: must be included before . */ ++/* Get mbstate_t, mbrtowc(), wcwidth(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ + #include "system.h" + #include "die.h" + #include "error.h" +@@ -324,6 +342,18 @@ + #include "xstrtol.h" + #include "xdectoint.h" + ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ ++#ifndef HAVE_DECL_WCWIDTH ++"this configure-time declaration test was not run" ++#endif ++#if !HAVE_DECL_WCWIDTH ++extern int wcwidth (); ++#endif ++ + /* The official name of this program (e.g., no 'g' prefix). */ + #define PROGRAM_NAME "pr" + +@@ -416,7 +446,20 @@ struct COLUMN + + typedef struct COLUMN COLUMN; + +-static int char_to_clump (char c); ++/* Funtion pointers to switch functions for single byte locale or for ++ multibyte locale. If multibyte functions do not exist in your sysytem, ++ these pointers always point the function for single byte locale. */ ++static void (*print_char) (char c); ++static int (*char_to_clump) (char c); ++ ++/* Functions for single byte locale. */ ++static void print_char_single (char c); ++static int char_to_clump_single (char c); ++ ++/* Functions for multibyte locale. */ ++static void print_char_multi (char c); ++static int char_to_clump_multi (char c); ++ + static bool read_line (COLUMN *p); + static bool print_page (void); + static bool print_stored (COLUMN *p); +@@ -428,6 +471,7 @@ static void add_line_number (COLUMN *p); + static void getoptnum (const char *n_str, int min, int *num, + const char *errfmt); + static void getoptarg (char *arg, char switch_char, char *character, ++ int *character_length, int *character_width, + int *number); + static void print_files (int number_of_files, char **av); + static void init_parameters (int number_of_files); +@@ -441,7 +485,6 @@ static void store_char (char c); + static void pad_down (unsigned int lines); + static void read_rest_of_line (COLUMN *p); + static void skip_read (COLUMN *p, int column_number); +-static void print_char (char c); + static void cleanup (void); + static void print_sep_string (void); + static void separator_string (const char *optarg_S); +@@ -453,7 +496,7 @@ static COLUMN *column_vector; + we store the leftmost columns contiguously in buff. + To print a line from buff, get the index of the first character + from line_vector[i], and print up to line_vector[i + 1]. */ +-static char *buff; ++static unsigned char *buff; + + /* Index of the position in buff where the next character + will be stored. */ +@@ -557,7 +600,7 @@ static int chars_per_column; + static bool untabify_input = false; + + /* (-e) The input tab character. */ +-static char input_tab_char = '\t'; ++static char input_tab_char[MB_LEN_MAX] = "\t"; + + /* (-e) Tabstops are at chars_per_tab, 2*chars_per_tab, 3*chars_per_tab, ... + where the leftmost column is 1. */ +@@ -567,7 +610,10 @@ static int chars_per_input_tab = 8; + static bool tabify_output = false; + + /* (-i) The output tab character. */ +-static char output_tab_char = '\t'; ++static char output_tab_char[MB_LEN_MAX] = "\t"; ++ ++/* (-i) The byte length of output tab character. */ ++static int output_tab_char_length = 1; + + /* (-i) The width of the output tab. */ + static int chars_per_output_tab = 8; +@@ -637,7 +683,13 @@ static int line_number; + static bool numbered_lines = false; + + /* (-n) Character which follows each line number. */ +-static char number_separator = '\t'; ++static char number_separator[MB_LEN_MAX] = "\t"; ++ ++/* (-n) The byte length of the character which follows each line number. */ ++static int number_separator_length = 1; ++ ++/* (-n) The character width of the character which follows each line number. */ ++static int number_separator_width = 0; + + /* (-n) line counting starts with 1st line of input file (not with 1st + line of 1st page printed). */ +@@ -690,6 +742,7 @@ static bool use_col_separator = false; + -a|COLUMN|-m is a 'space' and with the -J option a 'tab'. */ + static char const *col_sep_string = ""; + static int col_sep_length = 0; ++static int col_sep_width = 0; + static char *column_separator = (char *) " "; + static char *line_separator = (char *) "\t"; + +@@ -851,6 +904,13 @@ separator_string (const char *optarg_S) + integer_overflow (); + col_sep_length = len; + col_sep_string = optarg_S; ++ ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ col_sep_width = mbswidth (col_sep_string, 0); ++ else ++#endif ++ col_sep_width = col_sep_length; + } + + int +@@ -875,6 +935,21 @@ main (int argc, char **argv) + + atexit (close_stdout); + ++/* Define which functions are used, the ones for single byte locale or the ones ++ for multibyte locale. */ ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ print_char = print_char_multi; ++ char_to_clump = char_to_clump_multi; ++ } ++ else ++#endif ++ { ++ print_char = print_char_single; ++ char_to_clump = char_to_clump_single; ++ } ++ + n_files = 0; + file_names = (argc > 1 + ? xnmalloc (argc - 1, sizeof (char *)) +@@ -951,8 +1026,12 @@ main (int argc, char **argv) + break; + case 'e': + if (optarg) +- getoptarg (optarg, 'e', &input_tab_char, +- &chars_per_input_tab); ++ { ++ int dummy_length, dummy_width; ++ ++ getoptarg (optarg, 'e', input_tab_char, &dummy_length, ++ &dummy_width, &chars_per_input_tab); ++ } + /* Could check tab width > 0. */ + untabify_input = true; + break; +@@ -965,8 +1044,12 @@ main (int argc, char **argv) + break; + case 'i': + if (optarg) +- getoptarg (optarg, 'i', &output_tab_char, +- &chars_per_output_tab); ++ { ++ int dummy_width; ++ ++ getoptarg (optarg, 'i', output_tab_char, &output_tab_char_length, ++ &dummy_width, &chars_per_output_tab); ++ } + /* Could check tab width > 0. */ + tabify_output = true; + break; +@@ -984,8 +1067,8 @@ main (int argc, char **argv) + case 'n': + numbered_lines = true; + if (optarg) +- getoptarg (optarg, 'n', &number_separator, +- &chars_per_number); ++ getoptarg (optarg, 'n', number_separator, &number_separator_length, ++ &number_separator_width, &chars_per_number); + break; + case 'N': + skip_count = false; +@@ -1010,6 +1093,7 @@ main (int argc, char **argv) + /* Reset an additional input of -s, -S dominates -s */ + col_sep_string = ""; + col_sep_length = 0; ++ col_sep_width = 0; + use_col_separator = true; + if (optarg) + separator_string (optarg); +@@ -1166,10 +1250,45 @@ getoptnum (const char *n_str, int min, i + a number. */ + + static void +-getoptarg (char *arg, char switch_char, char *character, int *number) ++getoptarg (char *arg, char switch_char, char *character, int *character_length, ++ int *character_width, int *number) + { + if (!ISDIGIT (*arg)) +- *character = *arg++; ++ { ++#ifdef HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) /* for multibyte locale. */ ++ { ++ wchar_t wc; ++ size_t mblength; ++ int width; ++ mbstate_t state = {'\0'}; ++ ++ mblength = mbrtowc (&wc, arg, strnlen(arg, MB_LEN_MAX), &state); ++ ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ *character_length = 1; ++ *character_width = 1; ++ } ++ else ++ { ++ *character_length = (mblength < 1) ? 1 : mblength; ++ width = wcwidth (wc); ++ *character_width = (width < 0) ? 0 : width; ++ } ++ ++ strncpy (character, arg, *character_length); ++ arg += *character_length; ++ } ++ else /* for single byte locale. */ ++#endif ++ { ++ *character = *arg++; ++ *character_length = 1; ++ *character_width = 1; ++ } ++ } ++ + if (*arg) + { + long int tmp_long; +@@ -1191,6 +1310,11 @@ static void + init_parameters (int number_of_files) + { + int chars_used_by_number = 0; ++ int mb_len = 1; ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ mb_len = MB_LEN_MAX; ++#endif + + lines_per_body = lines_per_page - lines_per_header - lines_per_footer; + if (lines_per_body <= 0) +@@ -1228,7 +1352,7 @@ init_parameters (int number_of_files) + else + col_sep_string = column_separator; + +- col_sep_length = 1; ++ col_sep_length = col_sep_width = 1; + use_col_separator = true; + } + /* It's rather pointless to define a TAB separator with column +@@ -1258,11 +1382,11 @@ init_parameters (int number_of_files) + + TAB_WIDTH (chars_per_input_tab, chars_per_number); */ + + /* Estimate chars_per_text without any margin and keep it constant. */ +- if (number_separator == '\t') ++ if (number_separator[0] == '\t') + number_width = (chars_per_number + + TAB_WIDTH (chars_per_default_tab, chars_per_number)); + else +- number_width = chars_per_number + 1; ++ number_width = chars_per_number + number_separator_width; + + /* The number is part of the column width unless we are + printing files in parallel. */ +@@ -1271,7 +1395,7 @@ init_parameters (int number_of_files) + } + + int sep_chars, useful_chars; +- if (INT_MULTIPLY_WRAPV (columns - 1, col_sep_length, &sep_chars)) ++ if (INT_MULTIPLY_WRAPV (columns - 1, col_sep_width, &sep_chars)) + sep_chars = INT_MAX; + if (INT_SUBTRACT_WRAPV (chars_per_line - chars_used_by_number, sep_chars, + &useful_chars)) +@@ -1294,7 +1418,7 @@ init_parameters (int number_of_files) + We've to use 8 as the lower limit, if we use chars_per_default_tab = 8 + to expand a tab which is not an input_tab-char. */ + free (clump_buff); +- clump_buff = xmalloc (MAX (8, chars_per_input_tab)); ++ clump_buff = xmalloc (mb_len * MAX (8, chars_per_input_tab)); + } + + /* Open the necessary files, +@@ -1402,7 +1526,7 @@ init_funcs (void) + + /* Enlarge p->start_position of first column to use the same form of + padding_not_printed with all columns. */ +- h = h + col_sep_length; ++ h = h + col_sep_width; + + /* This loop takes care of all but the rightmost column. */ + +@@ -1436,7 +1560,7 @@ init_funcs (void) + } + else + { +- h = h_next + col_sep_length; ++ h = h_next + col_sep_width; + h_next = h + chars_per_column; + } + } +@@ -1727,9 +1851,9 @@ static void + align_column (COLUMN *p) + { + padding_not_printed = p->start_position; +- if (col_sep_length < padding_not_printed) ++ if (col_sep_width < padding_not_printed) + { +- pad_across_to (padding_not_printed - col_sep_length); ++ pad_across_to (padding_not_printed - col_sep_width); + padding_not_printed = ANYWHERE; + } + +@@ -2004,13 +2128,13 @@ store_char (char c) + /* May be too generous. */ + buff = X2REALLOC (buff, &buff_allocated); + } +- buff[buff_current++] = c; ++ buff[buff_current++] = (unsigned char) c; + } + + static void + add_line_number (COLUMN *p) + { +- int i; ++ int i, j; + char *s; + int num_width; + +@@ -2027,22 +2151,24 @@ add_line_number (COLUMN *p) + /* Tabification is assumed for multiple columns, also for n-separators, + but 'default n-separator = TAB' hasn't been given priority over + equal column_width also specified by POSIX. */ +- if (number_separator == '\t') ++ if (number_separator[0] == '\t') + { + i = number_width - chars_per_number; + while (i-- > 0) + (p->char_func) (' '); + } + else +- (p->char_func) (number_separator); ++ for (j = 0; j < number_separator_length; j++) ++ (p->char_func) (number_separator[j]); + } + else + /* To comply with POSIX, we avoid any expansion of default TAB + separator with a single column output. No column_width requirement + has to be considered. */ + { +- (p->char_func) (number_separator); +- if (number_separator == '\t') ++ for (j = 0; j < number_separator_length; j++) ++ (p->char_func) (number_separator[j]); ++ if (number_separator[0] == '\t') + output_position = POS_AFTER_TAB (chars_per_output_tab, + output_position); + } +@@ -2203,7 +2329,7 @@ print_white_space (void) + while (goal - h_old > 1 + && (h_new = POS_AFTER_TAB (chars_per_output_tab, h_old)) <= goal) + { +- putchar (output_tab_char); ++ fwrite (output_tab_char, sizeof(char), output_tab_char_length, stdout); + h_old = h_new; + } + while (++h_old <= goal) +@@ -2223,6 +2349,7 @@ print_sep_string (void) + { + char const *s = col_sep_string; + int l = col_sep_length; ++ int not_space_flag; + + if (separators_not_printed <= 0) + { +@@ -2234,6 +2361,7 @@ print_sep_string (void) + { + for (; separators_not_printed > 0; --separators_not_printed) + { ++ not_space_flag = 0; + while (l-- > 0) + { + /* 3 types of sep_strings: spaces only, spaces and chars, +@@ -2247,12 +2375,15 @@ print_sep_string (void) + } + else + { ++ not_space_flag = 1; + if (spaces_not_printed > 0) + print_white_space (); + putchar (*s++); +- ++output_position; + } + } ++ if (not_space_flag) ++ output_position += col_sep_width; ++ + /* sep_string ends with some spaces */ + if (spaces_not_printed > 0) + print_white_space (); +@@ -2280,7 +2411,7 @@ print_clump (COLUMN *p, int n, char *clu + required number of tabs and spaces. */ + + static void +-print_char (char c) ++print_char_single (char c) + { + if (tabify_output) + { +@@ -2304,6 +2435,74 @@ print_char (char c) + putchar (c); + } + ++#ifdef HAVE_MBRTOWC ++static void ++print_char_multi (char c) ++{ ++ static size_t mbc_pos = 0; ++ static char mbc[MB_LEN_MAX] = {'\0'}; ++ static mbstate_t state = {'\0'}; ++ mbstate_t state_bak; ++ wchar_t wc; ++ size_t mblength; ++ int width; ++ ++ if (tabify_output) ++ { ++ state_bak = state; ++ mbc[mbc_pos++] = c; ++ mblength = mbrtowc (&wc, mbc, mbc_pos, &state); ++ ++ while (mbc_pos > 0) ++ { ++ switch (mblength) ++ { ++ case (size_t)-2: ++ state = state_bak; ++ return; ++ ++ case (size_t)-1: ++ state = state_bak; ++ ++output_position; ++ putchar (mbc[0]); ++ memmove (mbc, mbc + 1, MB_CUR_MAX - 1); ++ --mbc_pos; ++ break; ++ ++ case 0: ++ mblength = 1; ++ ++ default: ++ if (wc == L' ') ++ { ++ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength); ++ --mbc_pos; ++ ++spaces_not_printed; ++ return; ++ } ++ else if (spaces_not_printed > 0) ++ print_white_space (); ++ ++ /* Nonprintables are assumed to have width 0, except L'\b'. */ ++ if ((width = wcwidth (wc)) < 1) ++ { ++ if (wc == L'\b') ++ --output_position; ++ } ++ else ++ output_position += width; ++ ++ fwrite (mbc, sizeof(char), mblength, stdout); ++ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength); ++ mbc_pos -= mblength; ++ } ++ } ++ return; ++ } ++ putchar (c); ++} ++#endif ++ + /* Skip to page PAGE before printing. + PAGE may be larger than total number of pages. */ + +@@ -2483,9 +2682,9 @@ read_line (COLUMN *p) + align_empty_cols = false; + } + +- if (col_sep_length < padding_not_printed) ++ if (col_sep_width < padding_not_printed) + { +- pad_across_to (padding_not_printed - col_sep_length); ++ pad_across_to (padding_not_printed - col_sep_width); + padding_not_printed = ANYWHERE; + } + +@@ -2555,7 +2754,7 @@ print_stored (COLUMN *p) + int i; + + int line = p->current_line++; +- char *first = &buff[line_vector[line]]; ++ unsigned char *first = &buff[line_vector[line]]; + /* FIXME + UMR: Uninitialized memory read: + * This is occurring while in: +@@ -2567,7 +2766,7 @@ print_stored (COLUMN *p) + xmalloc [xmalloc.c:94] + init_store_cols [pr.c:1648] + */ +- char *last = &buff[line_vector[line + 1]]; ++ unsigned char *last = &buff[line_vector[line + 1]]; + + pad_vertically = true; + +@@ -2586,9 +2785,9 @@ print_stored (COLUMN *p) + } + } + +- if (col_sep_length < padding_not_printed) ++ if (col_sep_width < padding_not_printed) + { +- pad_across_to (padding_not_printed - col_sep_length); ++ pad_across_to (padding_not_printed - col_sep_width); + padding_not_printed = ANYWHERE; + } + +@@ -2601,8 +2800,8 @@ print_stored (COLUMN *p) + if (spaces_not_printed == 0) + { + output_position = p->start_position + end_vector[line]; +- if (p->start_position - col_sep_length == chars_per_margin) +- output_position -= col_sep_length; ++ if (p->start_position - col_sep_width == chars_per_margin) ++ output_position -= col_sep_width; + } + + return true; +@@ -2621,7 +2820,7 @@ print_stored (COLUMN *p) + number of characters is 1.) */ + + static int +-char_to_clump (char c) ++char_to_clump_single (char c) + { + unsigned char uc = c; + char *s = clump_buff; +@@ -2631,10 +2830,10 @@ char_to_clump (char c) + int chars; + int chars_per_c = 8; + +- if (c == input_tab_char) ++ if (c == input_tab_char[0]) + chars_per_c = chars_per_input_tab; + +- if (c == input_tab_char || c == '\t') ++ if (c == input_tab_char[0] || c == '\t') + { + width = TAB_WIDTH (chars_per_c, input_position); + +@@ -2715,6 +2914,164 @@ char_to_clump (char c) + return chars; + } + ++#ifdef HAVE_MBRTOWC ++static int ++char_to_clump_multi (char c) ++{ ++ static size_t mbc_pos = 0; ++ static char mbc[MB_LEN_MAX] = {'\0'}; ++ static mbstate_t state = {'\0'}; ++ mbstate_t state_bak; ++ wchar_t wc; ++ size_t mblength; ++ int wc_width; ++ register char *s = clump_buff; ++ register int i, j; ++ char esc_buff[4]; ++ int width; ++ int chars; ++ int chars_per_c = 8; ++ ++ state_bak = state; ++ mbc[mbc_pos++] = c; ++ mblength = mbrtowc (&wc, mbc, mbc_pos, &state); ++ ++ width = 0; ++ chars = 0; ++ while (mbc_pos > 0) ++ { ++ switch (mblength) ++ { ++ case (size_t)-2: ++ state = state_bak; ++ return 0; ++ ++ case (size_t)-1: ++ state = state_bak; ++ mblength = 1; ++ ++ if (use_esc_sequence || use_cntrl_prefix) ++ { ++ width = +4; ++ chars = +4; ++ *s++ = '\\'; ++ sprintf (esc_buff, "%03o", (unsigned char) mbc[0]); ++ for (i = 0; i <= 2; ++i) ++ *s++ = (int) esc_buff[i]; ++ } ++ else ++ { ++ width += 1; ++ chars += 1; ++ *s++ = mbc[0]; ++ } ++ break; ++ ++ case 0: ++ mblength = 1; ++ /* Fall through */ ++ ++ default: ++ if (memcmp (mbc, input_tab_char, mblength) == 0) ++ chars_per_c = chars_per_input_tab; ++ ++ if (memcmp (mbc, input_tab_char, mblength) == 0 || c == '\t') ++ { ++ int width_inc; ++ ++ width_inc = TAB_WIDTH (chars_per_c, input_position); ++ width += width_inc; ++ ++ if (untabify_input) ++ { ++ for (i = width_inc; i; --i) ++ *s++ = ' '; ++ chars += width_inc; ++ } ++ else ++ { ++ for (i = 0; i < mblength; i++) ++ *s++ = mbc[i]; ++ chars += mblength; ++ } ++ } ++ else if ((wc_width = wcwidth (wc)) < 1) ++ { ++ if (use_esc_sequence) ++ { ++ for (i = 0; i < mblength; i++) ++ { ++ width += 4; ++ chars += 4; ++ *s++ = '\\'; ++ sprintf (esc_buff, "%03o", (unsigned char) mbc[i]); ++ for (j = 0; j <= 2; ++j) ++ *s++ = (int) esc_buff[j]; ++ } ++ } ++ else if (use_cntrl_prefix) ++ { ++ if (wc < 0200) ++ { ++ width += 2; ++ chars += 2; ++ *s++ = '^'; ++ *s++ = wc ^ 0100; ++ } ++ else ++ { ++ for (i = 0; i < mblength; i++) ++ { ++ width += 4; ++ chars += 4; ++ *s++ = '\\'; ++ sprintf (esc_buff, "%03o", (unsigned char) mbc[i]); ++ for (j = 0; j <= 2; ++j) ++ *s++ = (int) esc_buff[j]; ++ } ++ } ++ } ++ else if (wc == L'\b') ++ { ++ width += -1; ++ chars += 1; ++ *s++ = c; ++ } ++ else ++ { ++ width += 0; ++ chars += mblength; ++ for (i = 0; i < mblength; i++) ++ *s++ = mbc[i]; ++ } ++ } ++ else ++ { ++ width += wc_width; ++ chars += mblength; ++ for (i = 0; i < mblength; i++) ++ *s++ = mbc[i]; ++ } ++ } ++ memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength); ++ mbc_pos -= mblength; ++ } ++ ++ /* Too many backspaces must put us in position 0 -- never negative. */ ++ if (width < 0 && input_position == 0) ++ { ++ chars = 0; ++ input_position = 0; ++ } ++ else if (width < 0 && input_position <= -width) ++ input_position = 0; ++ else ++ input_position += width; ++ ++ return chars; ++} ++#endif ++ + /* We've just printed some files and need to clean up things before + looking for more options and printing the next batch of files. + +diff -Naurp coreutils-8.27-orig/src/sort.c coreutils-8.27/src/sort.c +--- coreutils-8.27-orig/src/sort.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/sort.c 2017-03-11 23:49:22.416505389 -0600 +@@ -29,6 +29,14 @@ + #include + #include + #include ++#if HAVE_WCHAR_H ++# include ++#endif ++/* Get isw* functions. */ ++#if HAVE_WCTYPE_H ++# include ++#endif ++ + #include "system.h" + #include "argmatch.h" + #include "die.h" +@@ -165,14 +173,39 @@ static int decimal_point; + /* Thousands separator; if -1, then there isn't one. */ + static int thousands_sep; + ++/* True if -f is specified. */ ++static bool folding; ++ + /* Nonzero if the corresponding locales are hard. */ + static bool hard_LC_COLLATE; +-#if HAVE_NL_LANGINFO ++#if HAVE_LANGINFO_CODESET + static bool hard_LC_TIME; + #endif + + #define NONZERO(x) ((x) != 0) + ++/* get a multibyte character's byte length. */ ++#define GET_BYTELEN_OF_CHAR(LIM, PTR, MBLENGTH, STATE) \ ++ do \ ++ { \ ++ wchar_t wc; \ ++ mbstate_t state_bak; \ ++ \ ++ state_bak = STATE; \ ++ mblength = mbrtowc (&wc, PTR, LIM - PTR, &STATE); \ ++ \ ++ switch (MBLENGTH) \ ++ { \ ++ case (size_t)-1: \ ++ case (size_t)-2: \ ++ STATE = state_bak; \ ++ /* Fall through. */ \ ++ case 0: \ ++ MBLENGTH = 1; \ ++ } \ ++ } \ ++ while (0) ++ + /* The kind of blanks for '-b' to skip in various options. */ + enum blanktype { bl_start, bl_end, bl_both }; + +@@ -346,13 +379,11 @@ static bool reverse; + they were read if all keys compare equal. */ + static bool stable; + +-/* If TAB has this value, blanks separate fields. */ +-enum { TAB_DEFAULT = CHAR_MAX + 1 }; +- +-/* Tab character separating fields. If TAB_DEFAULT, then fields are ++/* Tab character separating fields. If tab_length is 0, then fields are + separated by the empty string between a non-blank character and a blank + character. */ +-static int tab = TAB_DEFAULT; ++static char tab[MB_LEN_MAX + 1]; ++static size_t tab_length = 0; + + /* Flag to remove consecutive duplicate lines from the output. + Only the last of a sequence of equal lines will be output. */ +@@ -811,6 +842,46 @@ reap_all (void) + reap (-1); + } + ++/* Function pointers. */ ++static void ++(*inittables) (void); ++static char * ++(*begfield) (const struct line*, const struct keyfield *); ++static char * ++(*limfield) (const struct line*, const struct keyfield *); ++static void ++(*skipblanks) (char **ptr, char *lim); ++static int ++(*getmonth) (char const *, size_t, char **); ++static int ++(*keycompare) (const struct line *, const struct line *); ++static int ++(*numcompare) (const char *, const char *); ++ ++/* Test for white space multibyte character. ++ Set LENGTH the byte length of investigated multibyte character. */ ++#if HAVE_MBRTOWC ++static int ++ismbblank (const char *str, size_t len, size_t *length) ++{ ++ size_t mblength; ++ wchar_t wc; ++ mbstate_t state; ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ mblength = mbrtowc (&wc, str, len, &state); ++ ++ if (mblength == (size_t)-1 || mblength == (size_t)-2) ++ { ++ *length = 1; ++ return 0; ++ } ++ ++ *length = (mblength < 1) ? 1 : mblength; ++ return iswblank (wc) || wc == '\n'; ++} ++#endif ++ + /* Clean up any remaining temporary files. */ + + static void +@@ -1255,7 +1326,7 @@ zaptemp (char const *name) + free (node); + } + +-#if HAVE_NL_LANGINFO ++#if HAVE_LANGINFO_CODESET + + static int + struct_month_cmp (void const *m1, void const *m2) +@@ -1270,7 +1341,7 @@ struct_month_cmp (void const *m1, void c + /* Initialize the character class tables. */ + + static void +-inittables (void) ++inittables_uni (void) + { + size_t i; + +@@ -1282,7 +1353,7 @@ inittables (void) + fold_toupper[i] = toupper (i); + } + +-#if HAVE_NL_LANGINFO ++#if HAVE_LANGINFO_CODESET + /* If we're not in the "C" locale, read different names for months. */ + if (hard_LC_TIME) + { +@@ -1364,6 +1435,84 @@ specify_nmerge (int oi, char c, char con + xstrtol_fatal (e, oi, c, long_options, s); + } + ++#if HAVE_MBRTOWC ++static void ++inittables_mb (void) ++{ ++ int i, j, k, l; ++ char *name, *s, *lc_time, *lc_ctype; ++ size_t s_len, mblength; ++ char mbc[MB_LEN_MAX]; ++ wchar_t wc, pwc; ++ mbstate_t state_mb, state_wc; ++ ++ lc_time = setlocale (LC_TIME, ""); ++ if (lc_time) ++ lc_time = xstrdup (lc_time); ++ ++ lc_ctype = setlocale (LC_CTYPE, ""); ++ if (lc_ctype) ++ lc_ctype = xstrdup (lc_ctype); ++ ++ if (lc_time && lc_ctype) ++ /* temporarily set LC_CTYPE to match LC_TIME, so that we can convert ++ * the names of months to upper case */ ++ setlocale (LC_CTYPE, lc_time); ++ ++ for (i = 0; i < MONTHS_PER_YEAR; i++) ++ { ++ s = (char *) nl_langinfo (ABMON_1 + i); ++ s_len = strlen (s); ++ monthtab[i].name = name = (char *) xmalloc (s_len + 1); ++ monthtab[i].val = i + 1; ++ ++ memset (&state_mb, '\0', sizeof (mbstate_t)); ++ memset (&state_wc, '\0', sizeof (mbstate_t)); ++ ++ for (j = 0; j < s_len;) ++ { ++ if (!ismbblank (s + j, s_len - j, &mblength)) ++ break; ++ j += mblength; ++ } ++ ++ for (k = 0; j < s_len;) ++ { ++ mblength = mbrtowc (&wc, (s + j), (s_len - j), &state_mb); ++ assert (mblength != (size_t)-1 && mblength != (size_t)-2); ++ if (mblength == 0) ++ break; ++ ++ pwc = towupper (wc); ++ if (pwc == wc) ++ { ++ memcpy (mbc, s + j, mblength); ++ j += mblength; ++ } ++ else ++ { ++ j += mblength; ++ mblength = wcrtomb (mbc, pwc, &state_wc); ++ assert (mblength != (size_t)0 && mblength != (size_t)-1); ++ } ++ ++ for (l = 0; l < mblength; l++) ++ name[k++] = mbc[l]; ++ } ++ name[k] = '\0'; ++ } ++ qsort ((void *) monthtab, MONTHS_PER_YEAR, ++ sizeof (struct month), struct_month_cmp); ++ ++ if (lc_time && lc_ctype) ++ /* restore the original locales */ ++ setlocale (LC_CTYPE, lc_ctype); ++ ++ free (lc_ctype); ++ free (lc_time); ++} ++#endif ++ + /* Specify the amount of main memory to use when sorting. */ + static void + specify_sort_size (int oi, char c, char const *s) +@@ -1597,7 +1746,7 @@ buffer_linelim (struct buffer const *buf + by KEY in LINE. */ + + static char * +-begfield (struct line const *line, struct keyfield const *key) ++begfield_uni (const struct line *line, const struct keyfield *key) + { + char *ptr = line->text, *lim = ptr + line->length - 1; + size_t sword = key->sword; +@@ -1606,10 +1755,10 @@ begfield (struct line const *line, struc + /* The leading field separator itself is included in a field when -t + is absent. */ + +- if (tab != TAB_DEFAULT) ++ if (tab_length) + while (ptr < lim && sword--) + { +- while (ptr < lim && *ptr != tab) ++ while (ptr < lim && *ptr != tab[0]) + ++ptr; + if (ptr < lim) + ++ptr; +@@ -1635,11 +1784,70 @@ begfield (struct line const *line, struc + return ptr; + } + ++#if HAVE_MBRTOWC ++static char * ++begfield_mb (const struct line *line, const struct keyfield *key) ++{ ++ int i; ++ char *ptr = line->text, *lim = ptr + line->length - 1; ++ size_t sword = key->sword; ++ size_t schar = key->schar; ++ size_t mblength; ++ mbstate_t state; ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ if (tab_length) ++ while (ptr < lim && sword--) ++ { ++ while (ptr < lim && memcmp (ptr, tab, tab_length) != 0) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ if (ptr < lim) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ } ++ else ++ while (ptr < lim && sword--) ++ { ++ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ if (ptr < lim) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ } ++ ++ if (key->skipsblanks) ++ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ ++ for (i = 0; i < schar; i++) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ++ if (ptr + mblength > lim) ++ break; ++ else ++ ptr += mblength; ++ } ++ ++ return ptr; ++} ++#endif ++ + /* Return the limit of (a pointer to the first character after) the field + in LINE specified by KEY. */ + + static char * +-limfield (struct line const *line, struct keyfield const *key) ++limfield_uni (const struct line *line, const struct keyfield *key) + { + char *ptr = line->text, *lim = ptr + line->length - 1; + size_t eword = key->eword, echar = key->echar; +@@ -1654,10 +1862,10 @@ limfield (struct line const *line, struc + 'beginning' is the first character following the delimiting TAB. + Otherwise, leave PTR pointing at the first 'blank' character after + the preceding field. */ +- if (tab != TAB_DEFAULT) ++ if (tab_length) + while (ptr < lim && eword--) + { +- while (ptr < lim && *ptr != tab) ++ while (ptr < lim && *ptr != tab[0]) + ++ptr; + if (ptr < lim && (eword || echar)) + ++ptr; +@@ -1703,10 +1911,10 @@ limfield (struct line const *line, struc + */ + + /* Make LIM point to the end of (one byte past) the current field. */ +- if (tab != TAB_DEFAULT) ++ if (tab_length) + { + char *newlim; +- newlim = memchr (ptr, tab, lim - ptr); ++ newlim = memchr (ptr, tab[0], lim - ptr); + if (newlim) + lim = newlim; + } +@@ -1737,6 +1945,130 @@ limfield (struct line const *line, struc + return ptr; + } + ++#if HAVE_MBRTOWC ++static char * ++limfield_mb (const struct line *line, const struct keyfield *key) ++{ ++ char *ptr = line->text, *lim = ptr + line->length - 1; ++ size_t eword = key->eword, echar = key->echar; ++ int i; ++ size_t mblength; ++ mbstate_t state; ++ ++ if (echar == 0) ++ eword++; /* skip all of end field. */ ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ if (tab_length) ++ while (ptr < lim && eword--) ++ { ++ while (ptr < lim && memcmp (ptr, tab, tab_length) != 0) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ if (ptr < lim && (eword | echar)) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ } ++ else ++ while (ptr < lim && eword--) ++ { ++ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ if (ptr < lim) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ } ++ ++ ++# ifdef POSIX_UNSPECIFIED ++ /* Make LIM point to the end of (one byte past) the current field. */ ++ if (tab_length) ++ { ++ char *newlim, *p; ++ ++ newlim = NULL; ++ for (p = ptr; p < lim;) ++ { ++ if (memcmp (p, tab, tab_length) == 0) ++ { ++ newlim = p; ++ break; ++ } ++ ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ p += mblength; ++ } ++ } ++ else ++ { ++ char *newlim; ++ newlim = ptr; ++ ++ while (newlim < lim && ismbblank (newlim, lim - newlim, &mblength)) ++ newlim += mblength; ++ if (ptr < lim) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ptr += mblength; ++ } ++ while (newlim < lim && !ismbblank (newlim, lim - newlim, &mblength)) ++ newlim += mblength; ++ lim = newlim; ++ } ++# endif ++ ++ if (echar != 0) ++ { ++ /* If we're skipping leading blanks, don't start counting characters ++ * until after skipping past any leading blanks. */ ++ if (key->skipeblanks) ++ while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength)) ++ ptr += mblength; ++ ++ memset (&state, '\0', sizeof(mbstate_t)); ++ ++ /* Advance PTR by ECHAR (if possible), but no further than LIM. */ ++ for (i = 0; i < echar; i++) ++ { ++ GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state); ++ ++ if (ptr + mblength > lim) ++ break; ++ else ++ ptr += mblength; ++ } ++ } ++ ++ return ptr; ++} ++#endif ++ ++static void ++skipblanks_uni (char **ptr, char *lim) ++{ ++ while (*ptr < lim && blanks[to_uchar (**ptr)]) ++ ++(*ptr); ++} ++ ++#if HAVE_MBRTOWC ++static void ++skipblanks_mb (char **ptr, char *lim) ++{ ++ size_t mblength; ++ while (*ptr < lim && ismbblank (*ptr, lim - *ptr, &mblength)) ++ (*ptr) += mblength; ++} ++#endif ++ + /* Fill BUF reading from FP, moving buf->left bytes from the end + of buf->buf to the beginning first. If EOF is reached and the + file wasn't terminated by a newline, supply one. Set up BUF's line +@@ -1823,8 +2155,22 @@ fillbuf (struct buffer *buf, FILE *fp, c + else + { + if (key->skipsblanks) +- while (blanks[to_uchar (*line_start)]) +- line_start++; ++ { ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ size_t mblength; ++ while (line_start < line->keylim && ++ ismbblank (line_start, ++ line->keylim - line_start, ++ &mblength)) ++ line_start += mblength; ++ } ++ else ++#endif ++ while (blanks[to_uchar (*line_start)]) ++ line_start++; ++ } + line->keybeg = line_start; + } + } +@@ -1958,12 +2304,10 @@ find_unit_order (char const *number) + < K/k < M < G < T < P < E < Z < Y */ + + static int +-human_numcompare (char const *a, char const *b) ++human_numcompare (char *a, char *b) + { +- while (blanks[to_uchar (*a)]) +- a++; +- while (blanks[to_uchar (*b)]) +- b++; ++ skipblanks(&a, a + strlen(a)); ++ skipblanks(&b, b + strlen(b)); + + int diff = find_unit_order (a) - find_unit_order (b); + return (diff ? diff : strnumcmp (a, b, decimal_point, thousands_sep)); +@@ -1974,7 +2318,7 @@ human_numcompare (char const *a, char co + hideously fast. */ + + static int +-numcompare (char const *a, char const *b) ++numcompare_uni (const char *a, const char *b) + { + while (blanks[to_uchar (*a)]) + a++; +@@ -1984,6 +2328,25 @@ numcompare (char const *a, char const *b + return strnumcmp (a, b, decimal_point, thousands_sep); + } + ++#if HAVE_MBRTOWC ++static int ++numcompare_mb (const char *a, const char *b) ++{ ++ size_t mblength, len; ++ len = strlen (a); /* okay for UTF-8 */ ++ while (*a && ismbblank (a, len > MB_CUR_MAX ? MB_CUR_MAX : len, &mblength)) ++ { ++ a += mblength; ++ len -= mblength; ++ } ++ len = strlen (b); /* okay for UTF-8 */ ++ while (*b && ismbblank (b, len > MB_CUR_MAX ? MB_CUR_MAX : len, &mblength)) ++ b += mblength; ++ ++ return strnumcmp (a, b, decimal_point, thousands_sep); ++} ++#endif /* HAV_EMBRTOWC */ ++ + /* Work around a problem whereby the long double value returned by glibc's + strtold ("NaN", ...) contains uninitialized bits: clear all bytes of + A and B before calling strtold. FIXME: remove this function once +@@ -2034,7 +2397,7 @@ general_numcompare (char const *sa, char + Return 0 if the name in S is not recognized. */ + + static int +-getmonth (char const *month, char **ea) ++getmonth_uni (char const *month, size_t len, char **ea) + { + size_t lo = 0; + size_t hi = MONTHS_PER_YEAR; +@@ -2310,15 +2673,14 @@ debug_key (struct line const *line, stru + char saved = *lim; + *lim = '\0'; + +- while (blanks[to_uchar (*beg)]) +- beg++; ++ skipblanks (&beg, lim); + + char *tighter_lim = beg; + + if (lim < beg) + tighter_lim = lim; + else if (key->month) +- getmonth (beg, &tighter_lim); ++ getmonth (beg, lim-beg, &tighter_lim); + else if (key->general_numeric) + ignore_value (strtold (beg, &tighter_lim)); + else if (key->numeric || key->human_numeric) +@@ -2452,7 +2814,7 @@ key_warnings (struct keyfield const *gke + /* Warn about significant leading blanks. */ + bool implicit_skip = key_numeric (key) || key->month; + bool line_offset = key->eword == 0 && key->echar != 0; /* -k1.x,1.y */ +- if (!zero_width && !gkey_only && tab == TAB_DEFAULT && !line_offset ++ if (!zero_width && !gkey_only && !tab_length && !line_offset + && ((!key->skipsblanks && !implicit_skip) + || (!key->skipsblanks && key->schar) + || (!key->skipeblanks && key->echar))) +@@ -2510,11 +2872,87 @@ key_warnings (struct keyfield const *gke + error (0, 0, _("option '-r' only applies to last-resort comparison")); + } + ++#if HAVE_MBRTOWC ++static int ++getmonth_mb (const char *s, size_t len, char **ea) ++{ ++ char *month; ++ register size_t i; ++ register int lo = 0, hi = MONTHS_PER_YEAR, result; ++ char *tmp; ++ size_t wclength, mblength; ++ const char *pp; ++ const wchar_t *wpp; ++ wchar_t *month_wcs; ++ mbstate_t state; ++ ++ while (len > 0 && ismbblank (s, len, &mblength)) ++ { ++ s += mblength; ++ len -= mblength; ++ } ++ ++ if (len == 0) ++ return 0; ++ ++ if (SIZE_MAX - len < 1) ++ xalloc_die (); ++ ++ month = (char *) xnmalloc (len + 1, MB_CUR_MAX); ++ ++ pp = tmp = (char *) xnmalloc (len + 1, MB_CUR_MAX); ++ memcpy (tmp, s, len); ++ tmp[len] = '\0'; ++ wpp = month_wcs = (wchar_t *) xnmalloc (len + 1, sizeof (wchar_t)); ++ memset (&state, '\0', sizeof (mbstate_t)); ++ ++ wclength = mbsrtowcs (month_wcs, &pp, len + 1, &state); ++ if (wclength == (size_t)-1 || pp != NULL) ++ error (SORT_FAILURE, 0, _("Invalid multibyte input %s."), quote(s)); ++ ++ for (i = 0; i < wclength; i++) ++ { ++ month_wcs[i] = towupper(month_wcs[i]); ++ if (iswblank (month_wcs[i])) ++ { ++ month_wcs[i] = L'\0'; ++ break; ++ } ++ } ++ ++ mblength = wcsrtombs (month, &wpp, (len + 1) * MB_CUR_MAX, &state); ++ assert (mblength != (-1) && wpp == NULL); ++ ++ do ++ { ++ int ix = (lo + hi) / 2; ++ ++ if (strncmp (month, monthtab[ix].name, strlen (monthtab[ix].name)) < 0) ++ hi = ix; ++ else ++ lo = ix; ++ } ++ while (hi - lo > 1); ++ ++ result = (!strncmp (month, monthtab[lo].name, strlen (monthtab[lo].name)) ++ ? monthtab[lo].val : 0); ++ ++ if (ea && result) ++ *ea = (char*) s + strlen (monthtab[lo].name); ++ ++ free (month); ++ free (tmp); ++ free (month_wcs); ++ ++ return result; ++} ++#endif ++ + /* Compare two lines A and B trying every key in sequence until there + are no more keys or a difference is found. */ + + static int +-keycompare (struct line const *a, struct line const *b) ++keycompare_uni (const struct line *a, const struct line *b) + { + struct keyfield *key = keylist; + +@@ -2599,7 +3037,7 @@ keycompare (struct line const *a, struct + else if (key->human_numeric) + diff = human_numcompare (ta, tb); + else if (key->month) +- diff = getmonth (ta, NULL) - getmonth (tb, NULL); ++ diff = getmonth (ta, tlena, NULL) - getmonth (tb, tlenb, NULL); + else if (key->random) + diff = compare_random (ta, tlena, tb, tlenb); + else if (key->version) +@@ -2715,6 +3153,211 @@ keycompare (struct line const *a, struct + return key->reverse ? -diff : diff; + } + ++#if HAVE_MBRTOWC ++static int ++keycompare_mb (const struct line *a, const struct line *b) ++{ ++ struct keyfield *key = keylist; ++ ++ /* For the first iteration only, the key positions have been ++ precomputed for us. */ ++ char *texta = a->keybeg; ++ char *textb = b->keybeg; ++ char *lima = a->keylim; ++ char *limb = b->keylim; ++ ++ size_t mblength_a, mblength_b; ++ wchar_t wc_a, wc_b; ++ mbstate_t state_a, state_b; ++ ++ int diff = 0; ++ ++ memset (&state_a, '\0', sizeof(mbstate_t)); ++ memset (&state_b, '\0', sizeof(mbstate_t)); ++ /* Ignore keys with start after end. */ ++ if (a->keybeg - a->keylim > 0) ++ return 0; ++ ++ ++ /* Ignore and/or translate chars before comparing. */ ++# define IGNORE_CHARS(NEW_LEN, LEN, TEXT, COPY, WC, MBLENGTH, STATE) \ ++ do \ ++ { \ ++ wchar_t uwc; \ ++ char mbc[MB_LEN_MAX]; \ ++ mbstate_t state_wc; \ ++ \ ++ for (NEW_LEN = i = 0; i < LEN;) \ ++ { \ ++ mbstate_t state_bak; \ ++ \ ++ state_bak = STATE; \ ++ MBLENGTH = mbrtowc (&WC, TEXT + i, LEN - i, &STATE); \ ++ \ ++ if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1 \ ++ || MBLENGTH == 0) \ ++ { \ ++ if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1) \ ++ STATE = state_bak; \ ++ if (!ignore) \ ++ COPY[NEW_LEN++] = TEXT[i]; \ ++ i++; \ ++ continue; \ ++ } \ ++ \ ++ if (ignore) \ ++ { \ ++ if ((ignore == nonprinting && !iswprint (WC)) \ ++ || (ignore == nondictionary \ ++ && !iswalnum (WC) && !iswblank (WC))) \ ++ { \ ++ i += MBLENGTH; \ ++ continue; \ ++ } \ ++ } \ ++ \ ++ if (translate) \ ++ { \ ++ \ ++ uwc = towupper(WC); \ ++ if (WC == uwc) \ ++ { \ ++ memcpy (mbc, TEXT + i, MBLENGTH); \ ++ i += MBLENGTH; \ ++ } \ ++ else \ ++ { \ ++ i += MBLENGTH; \ ++ WC = uwc; \ ++ memset (&state_wc, '\0', sizeof (mbstate_t)); \ ++ \ ++ MBLENGTH = wcrtomb (mbc, WC, &state_wc); \ ++ assert (MBLENGTH != (size_t)-1 && MBLENGTH != 0); \ ++ } \ ++ \ ++ for (j = 0; j < MBLENGTH; j++) \ ++ COPY[NEW_LEN++] = mbc[j]; \ ++ } \ ++ else \ ++ for (j = 0; j < MBLENGTH; j++) \ ++ COPY[NEW_LEN++] = TEXT[i++]; \ ++ } \ ++ COPY[NEW_LEN] = '\0'; \ ++ } \ ++ while (0) ++ ++ /* Actually compare the fields. */ ++ ++ for (;;) ++ { ++ /* Find the lengths. */ ++ size_t lena = lima <= texta ? 0 : lima - texta; ++ size_t lenb = limb <= textb ? 0 : limb - textb; ++ ++ char enda IF_LINT (= 0); ++ char endb IF_LINT (= 0); ++ ++ char const *translate = key->translate; ++ bool const *ignore = key->ignore; ++ ++ if (ignore || translate) ++ { ++ if (SIZE_MAX - lenb - 2 < lena) ++ xalloc_die (); ++ char *copy_a = (char *) xnmalloc (lena + lenb + 2, MB_CUR_MAX); ++ char *copy_b = copy_a + lena * MB_CUR_MAX + 1; ++ size_t new_len_a, new_len_b; ++ size_t i, j; ++ ++ IGNORE_CHARS (new_len_a, lena, texta, copy_a, ++ wc_a, mblength_a, state_a); ++ IGNORE_CHARS (new_len_b, lenb, textb, copy_b, ++ wc_b, mblength_b, state_b); ++ texta = copy_a; textb = copy_b; ++ lena = new_len_a; lenb = new_len_b; ++ } ++ else ++ { ++ /* Use the keys in-place, temporarily null-terminated. */ ++ enda = texta[lena]; texta[lena] = '\0'; ++ endb = textb[lenb]; textb[lenb] = '\0'; ++ } ++ ++ if (key->random) ++ diff = compare_random (texta, lena, textb, lenb); ++ else if (key->numeric | key->general_numeric | key->human_numeric) ++ { ++ char savea = *lima, saveb = *limb; ++ ++ *lima = *limb = '\0'; ++ diff = (key->numeric ? numcompare (texta, textb) ++ : key->general_numeric ? general_numcompare (texta, textb) ++ : human_numcompare (texta, textb)); ++ *lima = savea, *limb = saveb; ++ } ++ else if (key->version) ++ diff = filevercmp (texta, textb); ++ else if (key->month) ++ diff = getmonth (texta, lena, NULL) - getmonth (textb, lenb, NULL); ++ else if (lena == 0) ++ diff = - NONZERO (lenb); ++ else if (lenb == 0) ++ diff = 1; ++ else if (hard_LC_COLLATE && !folding) ++ { ++ diff = xmemcoll0 (texta, lena + 1, textb, lenb + 1); ++ } ++ else ++ { ++ diff = memcmp (texta, textb, MIN (lena, lenb)); ++ if (diff == 0) ++ diff = lena < lenb ? -1 : lena != lenb; ++ } ++ ++ if (ignore || translate) ++ free (texta); ++ else ++ { ++ texta[lena] = enda; ++ textb[lenb] = endb; ++ } ++ ++ if (diff) ++ goto not_equal; ++ ++ key = key->next; ++ if (! key) ++ break; ++ ++ /* Find the beginning and limit of the next field. */ ++ if (key->eword != -1) ++ lima = limfield (a, key), limb = limfield (b, key); ++ else ++ lima = a->text + a->length - 1, limb = b->text + b->length - 1; ++ ++ if (key->sword != -1) ++ texta = begfield (a, key), textb = begfield (b, key); ++ else ++ { ++ texta = a->text, textb = b->text; ++ if (key->skipsblanks) ++ { ++ while (texta < lima && ismbblank (texta, lima - texta, &mblength_a)) ++ texta += mblength_a; ++ while (textb < limb && ismbblank (textb, limb - textb, &mblength_b)) ++ textb += mblength_b; ++ } ++ } ++ } ++ ++not_equal: ++ if (key && key->reverse) ++ return -diff; ++ else ++ return diff; ++} ++#endif ++ + /* Compare two lines A and B, returning negative, zero, or positive + depending on whether A compares less than, equal to, or greater than B. */ + +@@ -2742,7 +3385,7 @@ compare (struct line const *a, struct li + diff = - NONZERO (blen); + else if (blen == 0) + diff = 1; +- else if (hard_LC_COLLATE) ++ else if (hard_LC_COLLATE && !folding) + { + /* Note xmemcoll0 is a performance enhancement as + it will not unconditionally write '\0' after the +@@ -4139,6 +4782,7 @@ set_ordering (char const *s, struct keyf + break; + case 'f': + key->translate = fold_toupper; ++ folding = true; + break; + case 'g': + key->general_numeric = true; +@@ -4218,7 +4862,7 @@ main (int argc, char **argv) + initialize_exit_failure (SORT_FAILURE); + + hard_LC_COLLATE = hard_locale (LC_COLLATE); +-#if HAVE_NL_LANGINFO ++#if HAVE_LANGINFO_CODESET + hard_LC_TIME = hard_locale (LC_TIME); + #endif + +@@ -4239,6 +4883,29 @@ main (int argc, char **argv) + thousands_sep = -1; + } + ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ inittables = inittables_mb; ++ begfield = begfield_mb; ++ limfield = limfield_mb; ++ skipblanks = skipblanks_mb; ++ getmonth = getmonth_mb; ++ keycompare = keycompare_mb; ++ numcompare = numcompare_mb; ++ } ++ else ++#endif ++ { ++ inittables = inittables_uni; ++ begfield = begfield_uni; ++ limfield = limfield_uni; ++ skipblanks = skipblanks_uni; ++ getmonth = getmonth_uni; ++ keycompare = keycompare_uni; ++ numcompare = numcompare_uni; ++ } ++ + have_read_stdin = false; + inittables (); + +@@ -4513,13 +5180,34 @@ main (int argc, char **argv) + + case 't': + { +- char newtab = optarg[0]; +- if (! newtab) ++ char newtab[MB_LEN_MAX + 1]; ++ size_t newtab_length = 1; ++ strncpy (newtab, optarg, MB_LEN_MAX); ++ if (! newtab[0]) + die (SORT_FAILURE, 0, _("empty tab")); +- if (optarg[1]) ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ wchar_t wc; ++ mbstate_t state; ++ ++ memset (&state, '\0', sizeof (mbstate_t)); ++ newtab_length = mbrtowc (&wc, newtab, strnlen (newtab, ++ MB_LEN_MAX), ++ &state); ++ switch (newtab_length) ++ { ++ case (size_t) -1: ++ case (size_t) -2: ++ case 0: ++ newtab_length = 1; ++ } ++ } ++#endif ++ if (newtab_length == 1 && optarg[1]) + { + if (STREQ (optarg, "\\0")) +- newtab = '\0'; ++ newtab[0] = '\0'; + else + { + /* Provoke with 'sort -txx'. Complain about +@@ -4530,9 +5218,11 @@ main (int argc, char **argv) + quote (optarg)); + } + } +- if (tab != TAB_DEFAULT && tab != newtab) ++ if (tab_length && (tab_length != newtab_length ++ || memcmp (tab, newtab, tab_length) != 0)) + die (SORT_FAILURE, 0, _("incompatible tabs")); +- tab = newtab; ++ memcpy (tab, newtab, newtab_length); ++ tab_length = newtab_length; + } + break; + +@@ -4770,12 +5460,10 @@ main (int argc, char **argv) + sort (files, nfiles, outfile, nthreads); + } + +-#ifdef lint + if (files_from) + readtokens0_free (&tok); + else + free (files); +-#endif + + if (have_read_stdin && fclose (stdin) == EOF) + sort_die (_("close failed"), "-"); +diff -Naurp coreutils-8.27-orig/src/unexpand.c coreutils-8.27/src/unexpand.c +--- coreutils-8.27-orig/src/unexpand.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/unexpand.c 2017-03-11 23:49:06.758133530 -0600 +@@ -38,6 +38,9 @@ + #include + #include + #include ++ ++#include ++ + #include "system.h" + #include "die.h" + #include "xstrndup.h" +@@ -107,24 +110,47 @@ unexpand (void) + { + /* Input stream. */ + FILE *fp = next_file (NULL); ++ mb_file_t mbf; + + /* The array of pending blanks. In non-POSIX locales, blanks can + include characters other than spaces, so the blanks must be + stored, not merely counted. */ +- char *pending_blank; ++ mbf_char_t *pending_blank; ++ /* True if the starting locale is utf8. */ ++ bool using_utf_locale; ++ ++ /* True if the first file contains BOM header. */ ++ bool found_bom; ++ using_utf_locale=check_utf_locale(); + + if (!fp) + return; ++ mbf_init (mbf, fp); ++ found_bom=check_bom(fp,&mbf); + ++ if (using_utf_locale == false && found_bom == true) ++ { ++ /*try using some predefined locale */ ++ ++ if (set_utf_locale () != 0) ++ { ++ error (EXIT_FAILURE, errno, _("cannot set UTF-8 locale")); ++ } ++ } + /* The worst case is a non-blank character, then one blank, then a + tab stop, then MAX_COLUMN_WIDTH - 1 blanks, then a non-blank; so + allocate MAX_COLUMN_WIDTH bytes to store the blanks. */ +- pending_blank = xmalloc (max_column_width); ++ pending_blank = xmalloc (max_column_width * sizeof (mbf_char_t)); ++ ++ if (found_bom == true) ++ { ++ print_bom(); ++ } + + while (true) + { + /* Input character, or EOF. */ +- int c; ++ mbf_char_t c; + + /* If true, perform translations. */ + bool convert = true; +@@ -158,12 +184,44 @@ unexpand (void) + + do + { +- while ((c = getc (fp)) < 0 && (fp = next_file (fp))) +- continue; ++ while (true) { ++ mbf_getc (c, mbf); ++ if ((mb_iseof (c)) && (fp = next_file (fp))) ++ { ++ mbf_init (mbf, fp); ++ if (fp!=NULL) ++ { ++ if (check_bom(fp,&mbf)==true) ++ { ++ /*Not the first file - check BOM header*/ ++ if (using_utf_locale==false && found_bom==false) ++ { ++ /*BOM header in subsequent file but not in the first one. */ ++ error (EXIT_FAILURE, errno, _("combination of files with and without BOM header")); ++ } ++ } ++ else ++ { ++ if(using_utf_locale==false && found_bom==true) ++ { ++ /*First file conatined BOM header - locale was switched to UTF ++ /*all subsequent files should contain BOM. */ ++ error (EXIT_FAILURE, errno, _("combination of files with and without BOM header")); ++ } ++ } ++ } ++ continue; ++ } ++ else ++ { ++ break; ++ } ++ } ++ + + if (convert) + { +- bool blank = !! isblank (c); ++ bool blank = mb_isblank (c); + + if (blank) + { +@@ -180,16 +238,16 @@ unexpand (void) + if (next_tab_column < column) + die (EXIT_FAILURE, 0, _("input line is too long")); + +- if (c == '\t') ++ if (mb_iseq (c, '\t')) + { + column = next_tab_column; + + if (pending) +- pending_blank[0] = '\t'; ++ mb_setascii (&pending_blank[0], '\t'); + } + else + { +- column++; ++ column += mb_width (c); + + if (! (prev_blank && column == next_tab_column)) + { +@@ -197,13 +255,14 @@ unexpand (void) + will be replaced by tabs. */ + if (column == next_tab_column) + one_blank_before_tab_stop = true; +- pending_blank[pending++] = c; ++ mb_copy (&pending_blank[pending++], &c); + prev_blank = true; + continue; + } + + /* Replace the pending blanks by a tab or two. */ +- pending_blank[0] = c = '\t'; ++ mb_setascii (&c, '\t'); ++ mb_setascii (&pending_blank[0], '\t'); + } + + /* Discard pending blanks, unless it was a single +@@ -211,7 +270,7 @@ unexpand (void) + pending = one_blank_before_tab_stop; + } + } +- else if (c == '\b') ++ else if (mb_iseq (c, '\b')) + { + /* Go back one column, and force recalculation of the + next tab stop. */ +@@ -219,9 +278,9 @@ unexpand (void) + next_tab_column = column; + tab_index -= !!tab_index; + } +- else ++ else if (!mb_iseq (c, '\n')) + { +- column++; ++ column += mb_width (c); + if (!column) + die (EXIT_FAILURE, 0, _("input line is too long")); + } +@@ -229,8 +288,11 @@ unexpand (void) + if (pending) + { + if (pending > 1 && one_blank_before_tab_stop) +- pending_blank[0] = '\t'; +- if (fwrite (pending_blank, 1, pending, stdout) != pending) ++ mb_setascii (&pending_blank[0], '\t'); ++ ++ for (int n = 0; n < pending; ++n) ++ mb_putc (pending_blank[n], stdout); ++ if (ferror (stdout)) + die (EXIT_FAILURE, errno, _("write error")); + pending = 0; + one_blank_before_tab_stop = false; +@@ -240,16 +302,17 @@ unexpand (void) + convert &= convert_entire_line || blank; + } + +- if (c < 0) ++ if (mb_iseof (c)) + { + free (pending_blank); + return; + } + +- if (putchar (c) < 0) ++ mb_putc (c, stdout); ++ if (ferror (stdout)) + die (EXIT_FAILURE, errno, _("write error")); + } +- while (c != '\n'); ++ while (!mb_iseq (c, '\n')); + } + } + +diff -Naurp coreutils-8.27-orig/src/uniq.c coreutils-8.27/src/uniq.c +--- coreutils-8.27-orig/src/uniq.c 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/src/uniq.c 2017-03-11 23:47:13.098285938 -0600 +@@ -21,6 +21,17 @@ + #include + #include + ++/* Get mbstate_t, mbrtowc(). */ ++#if HAVE_WCHAR_H ++# include ++#endif ++ ++/* Get isw* functions. */ ++#if HAVE_WCTYPE_H ++# include ++#endif ++#include ++ + #include "system.h" + #include "argmatch.h" + #include "linebuffer.h" +@@ -32,9 +43,21 @@ + #include "stdio--.h" + #include "xmemcoll.h" + #include "xstrtol.h" +-#include "memcasecmp.h" ++#include "xmemcoll.h" + #include "quote.h" + ++/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC ++ installation; work around this configuration error. */ ++#if !defined MB_LEN_MAX || MB_LEN_MAX < 2 ++# define MB_LEN_MAX 16 ++#endif ++ ++/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */ ++#if HAVE_MBRTOWC && defined mbstate_t ++# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0) ++#endif ++ ++ + /* The official name of this program (e.g., no 'g' prefix). */ + #define PROGRAM_NAME "uniq" + +@@ -144,6 +167,10 @@ enum + GROUP_OPTION = CHAR_MAX + 1 + }; + ++/* Function pointers. */ ++static char * ++(*find_field) (struct linebuffer *line); ++ + static struct option const longopts[] = + { + {"count", no_argument, NULL, 'c'}, +@@ -260,7 +287,7 @@ size_opt (char const *opt, char const *m + return a pointer to the beginning of the line's field to be compared. */ + + static char * _GL_ATTRIBUTE_PURE +-find_field (struct linebuffer const *line) ++find_field_uni (struct linebuffer *line) + { + size_t count; + char const *lp = line->buffer; +@@ -280,6 +307,83 @@ find_field (struct linebuffer const *lin + return line->buffer + i; + } + ++#if HAVE_MBRTOWC ++ ++# define MBCHAR_TO_WCHAR(WC, MBLENGTH, LP, POS, SIZE, STATEP, CONVFAIL) \ ++ do \ ++ { \ ++ mbstate_t state_bak; \ ++ \ ++ CONVFAIL = 0; \ ++ state_bak = *STATEP; \ ++ \ ++ MBLENGTH = mbrtowc (&WC, LP + POS, SIZE - POS, STATEP); \ ++ \ ++ switch (MBLENGTH) \ ++ { \ ++ case (size_t)-2: \ ++ case (size_t)-1: \ ++ *STATEP = state_bak; \ ++ CONVFAIL++; \ ++ /* Fall through */ \ ++ case 0: \ ++ MBLENGTH = 1; \ ++ } \ ++ } \ ++ while (0) ++ ++static char * ++find_field_multi (struct linebuffer *line) ++{ ++ size_t count; ++ char *lp = line->buffer; ++ size_t size = line->length - 1; ++ size_t pos; ++ size_t mblength; ++ wchar_t wc; ++ mbstate_t *statep; ++ int convfail = 0; ++ ++ pos = 0; ++ statep = &(line->state); ++ ++ /* skip fields. */ ++ for (count = 0; count < skip_fields && pos < size; count++) ++ { ++ while (pos < size) ++ { ++ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); ++ ++ if (convfail || !(iswblank (wc) || wc == '\n')) ++ { ++ pos += mblength; ++ break; ++ } ++ pos += mblength; ++ } ++ ++ while (pos < size) ++ { ++ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); ++ ++ if (!convfail && (iswblank (wc) || wc == '\n')) ++ break; ++ ++ pos += mblength; ++ } ++ } ++ ++ /* skip fields. */ ++ for (count = 0; count < skip_chars && pos < size; count++) ++ { ++ MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail); ++ pos += mblength; ++ } ++ ++ return lp + pos; ++} ++#endif ++ + /* Return false if two strings OLD and NEW match, true if not. + OLD and NEW point not to the beginnings of the lines + but rather to the beginnings of the fields to compare. +@@ -288,6 +392,8 @@ find_field (struct linebuffer const *lin + static bool + different (char *old, char *new, size_t oldlen, size_t newlen) + { ++ char *copy_old, *copy_new; ++ + if (check_chars < oldlen) + oldlen = check_chars; + if (check_chars < newlen) +@@ -295,14 +401,103 @@ different (char *old, char *new, size_t + + if (ignore_case) + { +- /* FIXME: This should invoke strcoll somehow. */ +- return oldlen != newlen || memcasecmp (old, new, oldlen); ++ size_t i; ++ ++ copy_old = xmalloc (oldlen + 1); ++ copy_new = xmalloc (oldlen + 1); ++ ++ for (i = 0; i < oldlen; i++) ++ { ++ copy_old[i] = toupper (old[i]); ++ copy_new[i] = toupper (new[i]); ++ } ++ bool rc = xmemcoll (copy_old, oldlen, copy_new, newlen); ++ free (copy_old); ++ free (copy_new); ++ return rc; + } +- else if (hard_LC_COLLATE) +- return xmemcoll (old, oldlen, new, newlen) != 0; + else +- return oldlen != newlen || memcmp (old, new, oldlen); ++ { ++ copy_old = (char *)old; ++ copy_new = (char *)new; ++ } ++ ++ return xmemcoll (copy_old, oldlen, copy_new, newlen); ++ ++} ++ ++#if HAVE_MBRTOWC ++static int ++different_multi (const char *old, const char *new, size_t oldlen, size_t newlen, mbstate_t oldstate, mbstate_t newstate) ++{ ++ size_t i, j, chars; ++ const char *str[2]; ++ char *copy[2]; ++ size_t len[2]; ++ mbstate_t state[2]; ++ size_t mblength; ++ wchar_t wc, uwc; ++ mbstate_t state_bak; ++ ++ str[0] = old; ++ str[1] = new; ++ len[0] = oldlen; ++ len[1] = newlen; ++ state[0] = oldstate; ++ state[1] = newstate; ++ ++ for (i = 0; i < 2; i++) ++ { ++ copy[i] = xmalloc (len[i] + 1); ++ memset (copy[i], '\0', len[i] + 1); ++ ++ for (j = 0, chars = 0; j < len[i] && chars < check_chars; chars++) ++ { ++ state_bak = state[i]; ++ mblength = mbrtowc (&wc, str[i] + j, len[i] - j, &(state[i])); ++ ++ switch (mblength) ++ { ++ case (size_t)-1: ++ case (size_t)-2: ++ state[i] = state_bak; ++ /* Fall through */ ++ case 0: ++ mblength = 1; ++ break; ++ ++ default: ++ if (ignore_case) ++ { ++ uwc = towupper (wc); ++ ++ if (uwc != wc) ++ { ++ mbstate_t state_wc; ++ size_t mblen; ++ ++ memset (&state_wc, '\0', sizeof(mbstate_t)); ++ mblen = wcrtomb (copy[i] + j, uwc, &state_wc); ++ assert (mblen != (size_t)-1); ++ } ++ else ++ memcpy (copy[i] + j, str[i] + j, mblength); ++ } ++ else ++ memcpy (copy[i] + j, str[i] + j, mblength); ++ } ++ j += mblength; ++ } ++ copy[i][j] = '\0'; ++ len[i] = j; ++ } ++ int rc = xmemcoll (copy[0], len[0], copy[1], len[1]); ++ free (copy[0]); ++ free (copy[1]); ++ return rc; ++ + } ++#endif + + /* Output the line in linebuffer LINE to standard output + provided that the switches say it should be output. +@@ -367,19 +562,38 @@ check_file (const char *infile, const ch + char *prevfield IF_LINT ( = NULL); + size_t prevlen IF_LINT ( = 0); + bool first_group_printed = false; ++#if HAVE_MBRTOWC ++ mbstate_t prevstate; ++ ++ memset (&prevstate, '\0', sizeof (mbstate_t)); ++#endif + + while (!feof (stdin)) + { + char *thisfield; + size_t thislen; + bool new_group; ++#if HAVE_MBRTOWC ++ mbstate_t thisstate; ++#endif + + if (readlinebuffer_delim (thisline, stdin, delimiter) == 0) + break; + + thisfield = find_field (thisline); + thislen = thisline->length - 1 - (thisfield - thisline->buffer); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ thisstate = thisline->state; + ++ new_group = (prevline->length == 0 ++ || different_multi (thisfield, prevfield, ++ thislen, prevlen, ++ thisstate, prevstate)); ++ } ++ else ++#endif + new_group = (prevline->length == 0 + || different (thisfield, prevfield, thislen, prevlen)); + +@@ -397,6 +611,10 @@ check_file (const char *infile, const ch + SWAP_LINES (prevline, thisline); + prevfield = thisfield; + prevlen = thislen; ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ prevstate = thisstate; ++#endif + first_group_printed = true; + } + } +@@ -409,17 +627,26 @@ check_file (const char *infile, const ch + size_t prevlen; + uintmax_t match_count = 0; + bool first_delimiter = true; ++#if HAVE_MBRTOWC ++ mbstate_t prevstate; ++#endif + + if (readlinebuffer_delim (prevline, stdin, delimiter) == 0) + goto closefiles; + prevfield = find_field (prevline); + prevlen = prevline->length - 1 - (prevfield - prevline->buffer); ++#if HAVE_MBRTOWC ++ prevstate = prevline->state; ++#endif + + while (!feof (stdin)) + { + bool match; + char *thisfield; + size_t thislen; ++#if HAVE_MBRTOWC ++ mbstate_t thisstate = thisline->state; ++#endif + if (readlinebuffer_delim (thisline, stdin, delimiter) == 0) + { + if (ferror (stdin)) +@@ -428,6 +655,14 @@ check_file (const char *infile, const ch + } + thisfield = find_field (thisline); + thislen = thisline->length - 1 - (thisfield - thisline->buffer); ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ match = !different_multi (thisfield, prevfield, ++ thislen, prevlen, thisstate, prevstate); ++ } ++ else ++#endif + match = !different (thisfield, prevfield, thislen, prevlen); + match_count += match; + +@@ -460,6 +695,9 @@ check_file (const char *infile, const ch + SWAP_LINES (prevline, thisline); + prevfield = thisfield; + prevlen = thislen; ++#if HAVE_MBRTOWC ++ prevstate = thisstate; ++#endif + if (!match) + match_count = 0; + } +@@ -506,6 +744,19 @@ main (int argc, char **argv) + + atexit (close_stdout); + ++#if HAVE_MBRTOWC ++ if (MB_CUR_MAX > 1) ++ { ++ find_field = find_field_multi; ++ } ++ else ++#endif ++ { ++ find_field = find_field_uni; ++ } ++ ++ ++ + skip_chars = 0; + skip_fields = 0; + check_chars = SIZE_MAX; +diff -Naurp coreutils-8.27-orig/tests/expand/mb.sh coreutils-8.27/tests/expand/mb.sh +--- coreutils-8.27-orig/tests/expand/mb.sh 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/tests/expand/mb.sh 2017-03-11 23:49:06.759133489 -0600 +@@ -0,0 +1,183 @@ ++#!/bin/sh ++ ++# Copyright (C) 2012-2017 Free Software Foundation, Inc. ++ ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the GNU General Public License as published by ++# the Free Software Foundation, either version 3 of the License, or ++# (at your option) any later version. ++ ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++ ++# You should have received a copy of the GNU General Public License ++# along with this program. If not, see . ++ ++. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src ++print_ver_ expand ++ ++export LC_ALL=en_US.UTF-8 ++ ++#input containing multibyte characters ++cat <<\EOF > in || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++EOF ++env printf ' äöü\t. öüä. \tä xx\n' >> in || framework_failure_ ++ ++cat <<\EOF > exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#multiple files as an input ++cat <<\EOF >> exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++expand ./in ./in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#test characters with display widths != 1 ++env printf '12345678 ++e\t|ascii(1) ++\u00E9\t|composed(1) ++e\u0301\t|decomposed(1) ++\u3000\t|ideo-space(2) ++\uFF0D\t|full-hypen(2) ++' > in || framework_failure_ ++ ++env printf '12345678 ++e |ascii(1) ++\u00E9 |composed(1) ++e\u0301 |decomposed(1) ++\u3000 |ideo-space(2) ++\uFF0D |full-hypen(2) ++' > exp || framework_failure_ ++ ++expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#shouldn't fail with "input line too long" ++#when a line starts with a control character ++env printf '\n' > in || framework_failure_ ++ ++expand < in > out || fail=1 ++compare in out > /dev/null 2>&1 || fail=1 ++ ++#non-Unicode characters interspersed between Unicode ones ++env printf '12345678 ++\t\xFF| ++\xFF\t| ++\t\xFFä| ++ä\xFF\t| ++\tä\xFF| ++\xFF\tä| ++äbcdef\xFF\t| ++' > in || framework_failure_ ++ ++env printf '12345678 ++ \xFF| ++\xFF | ++ \xFFä| ++ä\xFF | ++ ä\xFF| ++\xFF ä| ++äbcdef\xFF | ++' > exp || framework_failure_ ++ ++expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++ ++ ++#BOM header test 1 ++printf "\xEF\xBB\xBF" > in; cat <<\EOF >> in || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++EOF ++env printf ' äöü\t. öüä. \tä xx\n' >> in || framework_failure_ ++ ++printf "\xEF\xBB\xBF" > exp; cat <<\EOF >> exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++ ++expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LANG=C expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LC_ALL=C expand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++ ++printf '\xEF\xBB\xBF' > in1; cat <<\EOF >> in1 || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++EOF ++env printf ' äöü\t. öüä. \tä xx\n' >> in1 || framework_failure_ ++ ++ ++printf '\xEF\xBB\xBF' > exp; cat <<\EOF >> exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++expand in1 in1 > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LANG=C expand in1 in1 > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LC_ALL=C expand in1 in1 > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++exit $fail +diff -Naurp coreutils-8.27-orig/tests/i18n/sort.sh coreutils-8.27/tests/i18n/sort.sh +--- coreutils-8.27-orig/tests/i18n/sort.sh 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/tests/i18n/sort.sh 2017-03-11 23:47:13.100285838 -0600 +@@ -0,0 +1,29 @@ ++#!/bin/sh ++# Verify sort's multi-byte support. ++ ++. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src ++print_ver_ sort ++ ++export LC_ALL=en_US.UTF-8 ++locale -k LC_CTYPE | grep -q "charmap.*UTF-8" \ ++ || skip_ "No UTF-8 locale available" ++ ++# Enable heap consistency checkng on older systems ++export MALLOC_CHECK_=2 ++ ++ ++# check buffer overflow issue due to ++# expanding multi-byte representation due to case conversion ++# https://bugzilla.suse.com/show_bug.cgi?id=928749 ++cat < exp ++. ++ɑ ++EOF ++cat < out || fail=1 ++. ++ɑ ++EOF ++compare exp out || { fail=1; cat out; } ++ ++ ++Exit $fail +diff -Naurp coreutils-8.27-orig/tests/local.mk coreutils-8.27/tests/local.mk +--- coreutils-8.27-orig/tests/local.mk 2017-02-28 22:25:37.000000000 -0600 ++++ coreutils-8.27/tests/local.mk 2017-03-11 23:47:38.072058253 -0600 +@@ -352,6 +352,8 @@ all_tests = \ + tests/misc/sort-discrim.sh \ + tests/misc/sort-files0-from.pl \ + tests/misc/sort-float.sh \ ++ tests/misc/sort-mb-tests.sh \ ++ tests/i18n/sort.sh \ + tests/misc/sort-h-thousands-sep.sh \ + tests/misc/sort-merge.pl \ + tests/misc/sort-merge-fdlimit.sh \ +@@ -544,6 +546,7 @@ all_tests = \ + tests/du/threshold.sh \ + tests/du/trailing-slash.sh \ + tests/du/two-args.sh \ ++ tests/expand/mb.sh \ + tests/id/gnu-zero-uids.sh \ + tests/id/no-context.sh \ + tests/id/context.sh \ +@@ -684,6 +687,7 @@ all_tests = \ + tests/touch/read-only.sh \ + tests/touch/relative.sh \ + tests/touch/trailing-slash.sh \ ++ tests/unexpand/mb.sh \ + $(all_root_tests) + + # See tests/factor/create-test.sh. +diff -Naurp coreutils-8.27-orig/tests/misc/cut.pl coreutils-8.27/tests/misc/cut.pl +--- coreutils-8.27-orig/tests/misc/cut.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/cut.pl 2017-03-11 23:47:13.100285838 -0600 +@@ -23,9 +23,11 @@ use strict; + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + +-my $mb_locale = $ENV{LOCALE_FR_UTF8}; ++my $mb_locale; ++# uncommented enable multibyte paths ++$mb_locale = $ENV{LOCALE_FR_UTF8}; + ! defined $mb_locale || $mb_locale eq 'none' +- and $mb_locale = 'C'; ++ and $mb_locale = 'C'; + + my $prog = 'cut'; + my $try = "Try '$prog --help' for more information.\n"; +@@ -240,6 +242,7 @@ if ($mb_locale ne 'C') + my @new_t = @$t; + my $test_name = shift @new_t; + ++ next if ($test_name =~ "newline-[12][0-9]"); + push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; + } + push @Tests, @new; +diff -Naurp coreutils-8.27-orig/tests/misc/expand.pl coreutils-8.27/tests/misc/expand.pl +--- coreutils-8.27-orig/tests/misc/expand.pl 2017-03-01 11:16:46.000000000 -0600 ++++ coreutils-8.27/tests/misc/expand.pl 2017-03-11 23:47:13.101285788 -0600 +@@ -27,6 +27,15 @@ my $prog = 'expand'; + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + ++#comment out next line to disable multibyte tests ++my $mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ ++my $prog = 'expand'; ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + my @Tests = + ( + ['t1', '--tabs=3', {IN=>"a\tb"}, {OUT=>"a b"}], +@@ -152,6 +161,8 @@ my @Tests = + ['trail9', '--tab=1,2 -t/5',{IN=>"\ta\tb\tc"}, {OUT=>" a b c"}], + + # Test errors ++ # FIXME: The following tests contain ‘quoting’ specific to LC_MESSAGES ++ # So we force LC_MESSAGES=C to make them pass. + ['e1', '--tabs="a"', {IN=>''}, {OUT=>''}, {EXIT=>1}, + {ERR => "$prog: tab size contains invalid character(s): 'a'\n"}], + ['e2', "-t $UINTMAX_OFLOW", {IN=>''}, {OUT=>''}, {EXIT=>1}, +@@ -168,6 +179,37 @@ my @Tests = + {ERR => "$prog: '/' specifier not at start of number: '/'\n"}], + ); + ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether expand is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ push @new, ["$test_name-mb", @new_t, {ENV => "LANG=$mb_locale LC_MESSAGES=C"}]; ++ } ++ push @Tests, @new; ++ } ++ ++ ++@Tests = triple_test \@Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +diff -Naurp coreutils-8.27-orig/tests/misc/fold.pl coreutils-8.27/tests/misc/fold.pl +--- coreutils-8.27-orig/tests/misc/fold.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/fold.pl 2017-03-11 23:47:13.101285788 -0600 +@@ -20,9 +20,18 @@ use strict; + + (my $program_name = $0) =~ s|.*/||; + ++my $prog = 'fold'; ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + ++# uncommented to enable multibyte paths ++my $mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ + my @Tests = + ( + ['s1', '-w2 -s', {IN=>"a\t"}, {OUT=>"a\n\t"}], +@@ -31,9 +40,48 @@ my @Tests = + ['s4', '-w4 -s', {IN=>"abc ef\n"}, {OUT=>"abc \nef\n"}], + ); + ++# Add _POSIX2_VERSION=199209 to the environment of each test ++# that uses an old-style option like +1. ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether fold is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ ++@Tests = triple_test \@Tests; ++ ++# Remember that triple_test creates from each test with exactly one "IN" ++# file two more tests (.p and .r suffix on name) corresponding to reading ++# input from a file and from a pipe. The pipe-reading test would fail ++# due to a race condition about 1 in 20 times. ++# Remove the IN_PIPE version of the "output-is-input" test above. ++# The others aren't susceptible because they have three inputs each. ++@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +-my $prog = 'fold'; + my $fail = run_tests ($program_name, $prog, \@Tests, $save_temps, $verbose); + exit $fail; +diff -Naurp coreutils-8.27-orig/tests/misc/join.pl coreutils-8.27/tests/misc/join.pl +--- coreutils-8.27-orig/tests/misc/join.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/join.pl 2017-03-11 23:47:13.102285737 -0600 +@@ -25,6 +25,15 @@ my $limits = getlimits (); + + my $prog = 'join'; + ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ ++my $mb_locale; ++#Comment out next line to disable multibyte tests ++$mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ + my $delim = chr 0247; + sub t_subst ($) + { +@@ -329,8 +338,49 @@ foreach my $t (@tv) + push @Tests, $new_ent; + } + ++# Add _POSIX2_VERSION=199209 to the environment of each test ++# that uses an old-style option like +1. ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether join is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ #Adjust the output some error messages including test_name for mb ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR}} ++ (@new_t)) ++ { ++ my $sub2 = {ERR_SUBST => "s/$test_name-mb/$test_name/"}; ++ push @new_t, $sub2; ++ push @$t, $sub2; ++ } ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ + @Tests = triple_test \@Tests; + ++#skip invalid-j-mb test, it is failing because of the format ++@Tests = grep {$_->[0] ne 'invalid-j-mb'} @Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +diff -Naurp coreutils-8.27-orig/tests/misc/sort-mb-tests.sh coreutils-8.27/tests/misc/sort-mb-tests.sh +--- coreutils-8.27-orig/tests/misc/sort-mb-tests.sh 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/tests/misc/sort-mb-tests.sh 2017-03-11 23:47:13.102285737 -0600 +@@ -0,0 +1,45 @@ ++#!/bin/sh ++# Verify sort's multi-byte support. ++ ++. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src ++print_ver_ sort ++ ++export LC_ALL=en_US.UTF-8 ++locale -k LC_CTYPE | grep -q "charmap.*UTF-8" \ ++ || skip_ "No UTF-8 locale available" ++ ++ ++cat < exp ++Banana＠5 ++Apple＠10 ++Citrus＠20 ++Cherry＠30 ++EOF ++ ++cat < out || fail=1 ++Apple＠10 ++Banana＠5 ++Citrus＠20 ++Cherry＠30 ++EOF ++ ++compare exp out || { fail=1; cat out; } ++ ++ ++cat < exp ++Citrus＠ＡＡ20＠＠5 ++Cherry＠ＡＡ30＠＠10 ++Apple＠ＡＡ10＠＠20 ++Banana＠ＡＡ5＠＠30 ++EOF ++ ++cat < out || fail=1 ++Apple＠ＡＡ10＠＠20 ++Banana＠ＡＡ5＠＠30 ++Citrus＠ＡＡ20＠＠5 ++Cherry＠ＡＡ30＠＠10 ++EOF ++ ++compare exp out || { fail=1; cat out; } ++ ++Exit $fail +diff -Naurp coreutils-8.27-orig/tests/misc/sort-merge.pl coreutils-8.27/tests/misc/sort-merge.pl +--- coreutils-8.27-orig/tests/misc/sort-merge.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/sort-merge.pl 2017-03-11 23:47:13.102285737 -0600 +@@ -26,6 +26,15 @@ my $prog = 'sort'; + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + ++my $mb_locale; ++# uncommented according to upstream commit enabling multibyte paths ++$mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + # three empty files and one that says 'foo' + my @inputs = (+(map{{IN=> {"empty$_"=> ''}}}1..3), {IN=> {foo=> "foo\n"}}); + +@@ -77,6 +86,39 @@ my @Tests = + {OUT=>$big_input}], + ); + ++# Add _POSIX2_VERSION=199209 to the environment of each test ++# that uses an old-style option like +1. ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether sort is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ next if ($test_name =~ "nmerge-."); ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ ++@Tests = triple_test \@Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +diff -Naurp coreutils-8.27-orig/tests/misc/sort.pl coreutils-8.27/tests/misc/sort.pl +--- coreutils-8.27-orig/tests/misc/sort.pl 2017-01-21 08:53:43.000000000 -0600 ++++ coreutils-8.27/tests/misc/sort.pl 2017-03-11 23:47:13.103285687 -0600 +@@ -24,10 +24,15 @@ my $prog = 'sort'; + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + +-my $mb_locale = $ENV{LOCALE_FR_UTF8}; ++my $mb_locale; ++#Comment out next line to disable multibyte tests ++$mb_locale = $ENV{LOCALE_FR_UTF8}; + ! defined $mb_locale || $mb_locale eq 'none' + and $mb_locale = 'C'; + ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + # Since each test is run with a file name and with redirected stdin, + # the name in the diagnostic is either the file name or "-". + # Normalize each diagnostic to use '-'. +@@ -423,6 +428,38 @@ foreach my $t (@Tests) + } + } + ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether sort is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ #disable several failing tests until investigation, disable all tests with envvars set ++ next if (grep {ref $_ eq 'HASH' && exists $_->{ENV}} (@new_t)); ++ next if ($test_name =~ "18g" or $test_name =~ "sort-numeric" or $test_name =~ "08[ab]" or $test_name =~ "03[def]" or $test_name =~ "h4" or $test_name =~ "n1" or $test_name =~ "2[01]a"); ++ next if ($test_name =~ "11[ab]"); # avoid FP: expected result differs to MB result due to collation rules. ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ + @Tests = triple_test \@Tests; + + # Remember that triple_test creates from each test with exactly one "IN" +@@ -432,6 +469,7 @@ foreach my $t (@Tests) + # Remove the IN_PIPE version of the "output-is-input" test above. + # The others aren't susceptible because they have three inputs each. + @Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; ++@Tests = grep {$_->[0] ne 'output-is-input-mb.p'} @Tests; + + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; +diff -Naurp coreutils-8.27-orig/tests/misc/unexpand.pl coreutils-8.27/tests/misc/unexpand.pl +--- coreutils-8.27-orig/tests/misc/unexpand.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/unexpand.pl 2017-03-11 23:47:13.103285687 -0600 +@@ -27,6 +27,14 @@ my $limits = getlimits (); + + my $prog = 'unexpand'; + ++# comment out next line to disable multibyte tests ++my $mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + my @Tests = + ( + ['a1', {IN=> ' 'x 1 ."y\n"}, {OUT=> ' 'x 1 ."y\n"}], +@@ -128,6 +136,37 @@ my @Tests = + ['ts2', '-t5,8', {IN=>"x\t \t y\n"}, {OUT=>"x\t\t y\n"}], + ); + ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether unexpand is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ next if ($test_name =~ 'b-1'); ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ ++@Tests = triple_test \@Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +diff -Naurp coreutils-8.27-orig/tests/misc/uniq.pl coreutils-8.27/tests/misc/uniq.pl +--- coreutils-8.27-orig/tests/misc/uniq.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/misc/uniq.pl 2017-03-11 23:47:13.103285687 -0600 +@@ -23,9 +23,17 @@ my $limits = getlimits (); + my $prog = 'uniq'; + my $try = "Try '$prog --help' for more information.\n"; + ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + # Turn off localization of executable's output. + @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3; + ++my $mb_locale; ++#Comment out next line to disable multibyte tests ++$mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ + # When possible, create a "-z"-testing variant of each test. + sub add_z_variants($) + { +@@ -262,6 +270,53 @@ foreach my $t (@Tests) + and push @$t, {ENV=>'_POSIX2_VERSION=199209'}; + } + ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether uniq is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ # In test #145, replace the each ‘...’ by '...'. ++ if ($test_name =~ "145") ++ { ++ my $sub = { ERR_SUBST => "s/‘([^’]+)’/'\$1'/g"}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ next if ( $test_name =~ "schar" ++ or $test_name =~ "^obs-plus" ++ or $test_name =~ "119"); ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ ++# Remember that triple_test creates from each test with exactly one "IN" ++# file two more tests (.p and .r suffix on name) corresponding to reading ++# input from a file and from a pipe. The pipe-reading test would fail ++# due to a race condition about 1 in 20 times. ++# Remove the IN_PIPE version of the "output-is-input" test above. ++# The others aren't susceptible because they have three inputs each. ++ ++@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; ++ + @Tests = add_z_variants \@Tests; + @Tests = triple_test \@Tests; + +diff -Naurp coreutils-8.27-orig/tests/pr/pr-tests.pl coreutils-8.27/tests/pr/pr-tests.pl +--- coreutils-8.27-orig/tests/pr/pr-tests.pl 2017-01-01 16:34:24.000000000 -0600 ++++ coreutils-8.27/tests/pr/pr-tests.pl 2017-03-11 23:47:13.103285687 -0600 +@@ -24,6 +24,15 @@ use strict; + my $prog = 'pr'; + my $normalize_strerror = "s/': .*/'/"; + ++my $mb_locale; ++#Uncomment the following line to enable multibyte tests ++$mb_locale = $ENV{LOCALE_FR_UTF8}; ++! defined $mb_locale || $mb_locale eq 'none' ++ and $mb_locale = 'C'; ++ ++my $try = "Try \`$prog --help' for more information.\n"; ++my $inval = "$prog: invalid byte, character or field list\n$try"; ++ + my @tv = ( + + # -b option is no longer an official option. But it's still working to +@@ -474,8 +483,48 @@ push @Tests, + {IN=>{2=>"a\n"}}, + {OUT=>"a\t\t\t\t \t\t\ta\n"} ]; + ++# Add _POSIX2_VERSION=199209 to the environment of each test ++# that uses an old-style option like +1. ++if ($mb_locale ne 'C') ++ { ++ # Duplicate each test vector, appending "-mb" to the test name and ++ # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we ++ # provide coverage for the distro-added multi-byte code paths. ++ my @new; ++ foreach my $t (@Tests) ++ { ++ my @new_t = @$t; ++ my $test_name = shift @new_t; ++ ++ # Depending on whether pr is multi-byte-patched, ++ # it emits different diagnostics: ++ # non-MB: invalid byte or field list ++ # MB: invalid byte, character or field list ++ # Adjust the expected error output accordingly. ++ if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval} ++ (@new_t)) ++ { ++ my $sub = {ERR_SUBST => 's/, character//'}; ++ push @new_t, $sub; ++ push @$t, $sub; ++ } ++ #temporarily skip some failing tests ++ next if ($test_name =~ "col-0" or $test_name =~ "col-inval" or $test_name =~ "asan1"); ++ push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}]; ++ } ++ push @Tests, @new; ++ } ++ + @Tests = triple_test \@Tests; + ++# Remember that triple_test creates from each test with exactly one "IN" ++# file two more tests (.p and .r suffix on name) corresponding to reading ++# input from a file and from a pipe. The pipe-reading test would fail ++# due to a race condition about 1 in 20 times. ++# Remove the IN_PIPE version of the "output-is-input" test above. ++# The others aren't susceptible because they have three inputs each. ++@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests; ++ + my $save_temps = $ENV{DEBUG}; + my $verbose = $ENV{VERBOSE}; + +diff -Naurp coreutils-8.27-orig/tests/unexpand/mb.sh coreutils-8.27/tests/unexpand/mb.sh +--- coreutils-8.27-orig/tests/unexpand/mb.sh 1969-12-31 18:00:00.000000000 -0600 ++++ coreutils-8.27/tests/unexpand/mb.sh 2017-03-11 23:49:06.759133489 -0600 +@@ -0,0 +1,172 @@ ++#!/bin/sh ++ ++# Copyright (C) 2012-2017 Free Software Foundation, Inc. ++ ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the GNU General Public License as published by ++# the Free Software Foundation, either version 3 of the License, or ++# (at your option) any later version. ++ ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++ ++# You should have received a copy of the GNU General Public License ++# along with this program. If not, see . ++ ++. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src ++print_ver_ unexpand ++ ++export LC_ALL=en_US.UTF-8 ++ ++#input containing multibyte characters ++cat > in <<\EOF ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++cat > exp <<\EOF ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++unexpand -a < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++ ++#multiple files as an input ++cat >> exp <<\EOF ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++ ++unexpand -a ./in ./in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#test characters with a display width larger than 1 ++ ++env printf '12345678 ++e |ascii(1) ++\u00E9 |composed(1) ++e\u0301 |decomposed(1) ++\u3000 |ideo-space(2) ++\uFF0D |full-hypen(2) ++' > in || framework_failure_ ++ ++env printf '12345678 ++e\t|ascii(1) ++\u00E9\t|composed(1) ++e\u0301\t|decomposed(1) ++\u3000\t|ideo-space(2) ++\uFF0D\t|full-hypen(2) ++' > exp || framework_failure_ ++ ++unexpand -a < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#test input where a blank of width > 1 is not being substituted ++in="$(LC_ALL=en_US.UTF-8 printf ' \u3000 ö ü ß')" ++exp=' 　 ö ü ß' ++ ++unexpand -a < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#non-Unicode characters interspersed between Unicode ones ++env printf '12345678 ++ \xFF| ++\xFF | ++ \xFFä| ++ä\xFF | ++ ä\xFF| ++\xFF ä| ++äbcdef\xFF | ++' > in || framework_failure_ ++ ++env printf '12345678 ++\t\xFF| ++\xFF\t| ++\t\xFFä| ++ä\xFF\t| ++\tä\xFF| ++\xFF\tä| ++äbcdef\xFF\t| ++' > exp || framework_failure_ ++ ++unexpand -a < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++#BOM header test 1 ++printf "\xEF\xBB\xBF" > in; cat <<\EOF >> in || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++env printf ' äöü\t. öüä. \tä xx\n' >> in || framework_failure_ ++ ++printf "\xEF\xBB\xBF" > exp; cat <<\EOF >> exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++unexpand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LANG=C unexpand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LC_ALL=C unexpand < in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++ ++printf "\xEF\xBB\xBF" > exp; cat <<\EOF >> exp || framework_failure_ ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++1234567812345678123456781 ++. . . . ++a b c d ++. . . . ++ä ö ü ß ++. . . . ++ äöü . öüä. ä xx ++EOF ++ ++ ++unexpand in in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LANG=C unexpand in in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 ++ ++LC_ALL=C unexpand in in > out || fail=1 ++compare exp out > /dev/null 2>&1 || fail=1 diff -r c3c324c07d3f -r 90a5eb560fd6 coreutils/stuff/patches/coreutils-fix-po.patch --- a/coreutils/stuff/patches/coreutils-fix-po.patch Wed Feb 21 18:10:55 2018 +0200 +++ b/coreutils/stuff/patches/coreutils-fix-po.patch Wed Feb 21 19:48:17 2018 +0200 @@ -20,8 +20,8 @@ " N-M fra N'te til M'te (inklusive) byte, tegn eller felt\n" --- a/po/sl.po +++ b/po/sl.po -@@ -2641,7 +2641,7 @@ - #: src/csplit.c:1299 +@@ -2579,7 +2579,7 @@ + #: src/csplit.c:1298 #, c-format msgid "invalid conversion specifier in suffix: \\%.3o" -msgstr "neveljavno določilo pretvorbe v priponi: \\\\%.3o" diff -r c3c324c07d3f -r 90a5eb560fd6 coreutils/stuff/patches/series --- a/coreutils/stuff/patches/series Wed Feb 21 18:10:55 2018 +0200 +++ b/coreutils/stuff/patches/series Wed Feb 21 19:48:17 2018 +0200 @@ -1,3 +1,7 @@ -coreutils-8.25-i18n-2.patch +# LFS: Coreutils Internationalization Fixes Patch +#coreutils-8.29-i18n-1.patch + +# SliTaz: show extended info touching CPU via uname uname.u +# SliTaz: fix translations, especially deprecated symbol '\v' coreutils-fix-po.patch diff -r c3c324c07d3f -r 90a5eb560fd6 diffutils/receipt --- a/diffutils/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/diffutils/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,24 +1,22 @@ # SliTaz package receipt v2. PACKAGE="diffutils" -VERSION="3.5" +VERSION="3.6" CATEGORY="development" SHORT_DESC="Show differences between two files" MAINTAINER="pascal.bellard@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/diffutils/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/diffutils.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 e2fsprogs/receipt --- a/e2fsprogs/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/e2fsprogs/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,25 +1,23 @@ # SliTaz package receipt v2. PACKAGE="e2fsprogs" -VERSION="1.43.4" +VERSION="1.43.5" CATEGORY="base-system" SHORT_DESC="Filesystem utilities for use with ext2 and ext3" MAINTAINER="pankso@slitaz.org" LICENSE="GPL2" WEB_SITE="http://e2fsprogs.sourceforge.net/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/e2fsprogs.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="$SF_MIRROR/$PACKAGE/$TARBALL" +BUILD_DEPENDS_arm=" " BUILD_DEPENDS="util-linux-uuid-dev util-linux-blkid-dev gettext attr-dev \ acl-dev texinfo" -BUILD_DEPENDS_arm=" " -SPLIT="libcomerr libcomerr-dev e2fsprogs-dev e2fsprogs-fsck" +SPLIT="libcomerr-dev e2fsprogs-dev e2fsprogs-fsck libcomerr e2fsprogs" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { sed -i -e 's:\[\.-\]::' tests/filter.sed mkdir ../build && cd ../build @@ -33,7 +31,7 @@ --disable-uuidd \ --disable-fsck \ $CONFIGURE_ARGS && - make && make install && make install-libs + make && make install && make install-libs || return 1 for i in libcom_err libe2p libext2fs libss; do chmod u+w $install/usr/lib/$i.a @@ -45,7 +43,7 @@ $install/usr/share/info/libext2fs.info # Additional documentation - makeinfo -o $src/doc/com_err.info $src/lib/et/com_err.texinfo + makeinfo -o $src/doc/com_err.info $src/lib/et/com_err.texinfo install -m644 $src/doc/com_err.info $install/usr/share/info install-info --dir-file=$install/usr/share/info/dir \ $install/usr/share/info/com_err.info @@ -55,43 +53,36 @@ msgfmt $src/po/de.po -o $src/po/de.gmo } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in - e2fsprogs) - copy bin/ sbin/ etc/ *.so* - find $fs -name '*com_err*' -delete - # Remove fsck provided by Busybox. - rm $fs/sbin/fsck* - DEPENDS="libcomerr util-linux-blkid util-linux-uuid" + libcomerr-dev) + copy libcom_err.a com_err.h com_err.pc + CAT="development|libcomerr development files" + DEPENDS="pkg-config" ;; e2fsprogs-dev) - copy @dev *.awk *.sed - find $fs \( -name com_err.h -o -name libcom_err.a -o -name com_err.pc \) -delete + copy @dev *.sed *.awk @rm DEPENDS="libcomerr-dev pkg-config" ;; + e2fsprogs-fsck) + copy sbin/fsck.* + CAT="system-tools|fsck tools" + DEPENDS="e2fsprogs libcomerr util-linux-blkid util-linux-uuid" + ;; libcomerr) copy libcom_err.so* CAT="base-system|libcomerr library" DEPENDS=" " ;; - libcomerr-dev) - copy libcom_err.a com_err.h com_err.pc - CAT="development|libcomerr development files" - DEPENDS="pkg-config" - ;; - e2fsprogs-fsck) - copy fsck.* - rm -r $fs/usr - CAT="system-tools|fsck tools" - DEPENDS="e2fsprogs libcomerr util-linux-blkid util-linux-uuid" + e2fsprogs) + copy @std @rm + DEPENDS="libcomerr util-linux-blkid util-linux-uuid" + TAGS="LFS" ;; esac } # Overlap busybox -pre_install_e2fsprogs() -{ +pre_install_e2fsprogs() { rm -f $1/sbin/tune2fs } diff -r c3c324c07d3f -r 90a5eb560fd6 eudev/receipt --- a/eudev/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/eudev/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -8,7 +8,6 @@ LICENSE="GPL2" WEB_SITE="http://dev.gentoo.org/~blueness/eudev/" LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/eudev.html" -PROVIDE="udev" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="${WEB_SITE}$TARBALL" @@ -17,9 +16,7 @@ BUILD_DEPENDS="gobject-introspection-dev util-linux-blkid-dev gperf kmod-dev" SPLIT="eudev-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { sed -r -i 's|/usr(/bin/test)|\1|' test/udev-test.pl sed -i '/keyboard_lookup_key/d' src/udev/udev-builtin-keyboard.c cat > config.cache <<"EOF" @@ -58,14 +55,13 @@ # in /lib/udev/rules.d. } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in eudev) copy @std DEPENDS="kmod liblzma util-linux-blkid util-linux-uuid zlib dbus" PROVIDE="udev" + TAGS="LFS" ;; eudev-dev) copy @dev diff -r c3c324c07d3f -r 90a5eb560fd6 expat/receipt --- a/expat/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/expat/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,36 +1,34 @@ # SliTaz package receipt v2. PACKAGE="expat" -VERSION="2.2.0" +VERSION="2.2.3" CATEGORY="x-window" SHORT_DESC="XML parsing C library" MAINTAINER="pankso@slitaz.org" LICENSE="MIT" WEB_SITE="https://libexpat.github.io/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/expat.html" TARBALL="$PACKAGE-$VERSION.tar.bz2" WGET_URL="$SF_MIRROR/$PACKAGE/$TARBALL" SPLIT="expat-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure \ --disable-static \ $CONFIGURE_ARGS && - make && make DESTDIR=$install install + make && make DESTDIR=$install install || return 1 - docdir="$install/usr/share/doc/expat-$VERSION" - mkdir -p $docdir; cp doc/* $docdir; rm $docdir/*.1 + cook_pick_docs doc/*.html doc/*.css doc/*.png } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in - expat) copy @std ;; - expat-dev) copy @dev ;; + expat) + copy @std + TAGS="LFS" + ;; + *-dev) copy @dev;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 file/receipt --- a/file/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/file/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,7 +7,7 @@ MAINTAINER="erjo@slitaz.org" LICENSE="BSD" WEB_SITE="https://github.com/file/file" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/file.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="https://github.com/file/file/archive/FILE${VERSION/./_}.tar.gz" @@ -16,30 +16,31 @@ BUILD_DEPENDS_arm="" SPLIT="libmagic libmagic-dev python-magic" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { autoreconf -f -i && - ./configure $CONFIGURE_ARGS && make && make install + ./configure \ + --libdir=/usr/lib \ + $CONFIGURE_ARGS && + make && make install || return 1 + + rmdir --ignore-fail-on-non-empty $install/usr/share/man/man5/ cd python python setup.py build && - python setup.py install --root=$DESTDIR + python setup.py install --root=$install } # Be sure it as cross compile. -testsuite() -{ +testsuite() { readelf -h $install/usr/bin/file } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in file) copy file magic.mgc DEPENDS="zlib libmagic" + TAGS="LFS" ;; libmagic) copy *.so* diff -r c3c324c07d3f -r 90a5eb560fd6 findutils/receipt --- a/findutils/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/findutils/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,13 +7,12 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/findutils/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/findutils.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure \ --localstatedir=/var/lib/locate \ $CONFIGURE_ARGS && @@ -21,9 +20,8 @@ make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std DEPENDS="glibc-base slitaz-base-files" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 flex/receipt --- a/flex/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/flex/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,33 +7,31 @@ MAINTAINER="pankso@slitaz.org" LICENSE="BSD" WEB_SITE="https://github.com/westes/flex" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/flex.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="$WEB_SITE/releases/download/v$VERSION/$TARBALL" +TARBALL_MD5="2882e3179748cc9f9c23ec593d6adc8d" BUILD_DEPENDS="bison gettext help2man indent" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { case "$ARCH" in arm*) export ac_cv_func_malloc_0_nonnull=yes export ac_cv_func_realloc_0_nonnull=yes esac - ./configure \ - --docdir=/usr/share/doc/flex-$VERSION \ - $CONFIGURE_ARGS && - make && make install + # Fix a problem introduced with glibc-2.26 + sed -i "/math.h/a #include " src/flexdef.h + + ./configure $CONFIGURE_ARGS && make && make install || return 1 ln -s flex $install/usr/bin/lex } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std @dev DEPENDS="bison" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 gawk/receipt --- a/gawk/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/gawk/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,7 +7,7 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="http://www.gnu.org/software/gawk/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/gawk.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" @@ -16,25 +16,23 @@ SPLIT="gawk-dev" # Rules to compile & install the temporary toolchain. -cook_tmp_toolchain() -{ +cook_tmp_toolchain() { cd $src ./configure && make && make install } -# Rules to configure and make the package. -compile_rules() -{ - ./configure $CONFIGURE_ARGS && make && make install +compile_rules() { + ./configure $CONFIGURE_ARGS && make && make install || return 1 + + cook_pick_docs doc/awkforai.txt doc/*.eps doc/*.pdf doc/*.jpg } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in gawk) copy @std DEPENDS="readline" + TAGS="LFS" ;; *-dev) copy @dev;; esac diff -r c3c324c07d3f -r 90a5eb560fd6 gcc/receipt --- a/gcc/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/gcc/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,87 +1,99 @@ # SliTaz package receipt v2. +# Note, currently first pass is broken. Sorry. PACKAGE="gcc" -VERSION="6.3.0" +VERSION="7.3.0" CATEGORY="development" SHORT_DESC="The GNU Compiler Collection" MAINTAINER="pankso@slitaz.org" LICENSE="GPL2" WEB_SITE="http://gcc.gnu.org/" -#HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/gcc.html" -TARBALL="$PACKAGE-$VERSION.tar.bz2" -WGET_URL="ftp://gcc.gnu.org/pub/gcc/releases/gcc-$VERSION/$TARBALL" +TARBALL="$PACKAGE-$VERSION.tar.xz" +WGET_URL="http://ftp.gnu.org/gnu/gcc/gcc-$VERSION/$TARBALL" BUILD_DEPENDS="mpc-library-dev mpfr-dev gmp-dev elfutils-dev zlib-dev" SPLIT="gcc-lib-base gcc-lib-math libgfortran gfortran libgomp libobjc \ libatomic gcc" +if [ "$2" == "--first-pass" -o "$3" == "--first-pass" ]; then + STAGE='first' + SPLIT='' +fi + compile_rules() { # Handle cross compilation (native i486/ARM gcc) case "$ARCH" in i?86) ARCH_ARGS="--with-tune=$ARCH" pkgversion="SliTaz" - languages="c,c++,objc,fortran" ;; + languages="c,c++,objc,fortran" + sed -i 's/^T_CFLAGS =$/& -fomit-frame-pointer/' gcc/Makefile.in + ;; + x86_64) + ARCH_ARGS="" + pkgversion="SliTaz64" + languages="c,c++,objc,fortran" + sed -i.orig '/m64=/ s|lib64|lib|' gcc/config/i386/t-linux64 + ;; arm*) + ARCH_ARGS="" pkgversion="SliTaz ARM" - languages="c,c++" ;; + languages="c,c++" + ;; esac - # Package slitaz-toolchain use 'cook --options' when rebuilding - # the full SliTaz toolchain. - [ "$2" == "--first-pass" ] && opt=$2 - [ "$3" == "--first-pass" ] && opt=$3 - # Use libiberty.a from binutils. sed -i 's/install_to_$(INSTALL_DEST) //' libiberty/Makefile.in || return 1 - case "$ARCH" in - i?86) sed -i 's/^T_CFLAGS =$/& -fomit-frame-pointer/' gcc/Makefile.in ;; - esac + case "$STAGE" in + first) + echo "cook: configure GCC for: toolchain first pass" - #sed -i 's@\./fixinc\.sh@-c true@' gcc/Makefile.in - - mkdir -p ../gcc-build && cd ../gcc-build - - # This is the default GCC and we want a native build to cross compile after. - # SliTaz target i486 host so we need a native i486 build for GCC. - case "$opt" in - --first-pass) - # Used by slitaz-toolchain when rebuilding the full toolchain. - echo "cook: configure GCC for: toolchain first pass" - $src/configure \ - --libexecdir=/usr/lib \ + mkdir build + cd build + ../configure \ + --with-newlib \ + --without-headers \ --disable-nls \ - --enable-shared \ + --disable-shared \ + --disable-multilib \ + --disable-decimal-float \ + --disable-threads \ + --disable-libatomic \ + --disable-libgomp \ + --disable-libmpx \ + --disable-libquadmath \ + --disable-libssp \ + --disable-libvtv \ + --disable-libstdcxx \ --enable-languages=c,c++ \ - --disable-libstdcxx-pch \ - --enable-__cxa_atexit \ - --enable-clocale=gnu \ - --enable-threads=posix \ - --disable-bootstrap \ - ${CONFIGURE_ARGS} + $CONFIGURE_ARGS ;; *) - # Used by default to produce a full featured X86 GCC compiler. echo "cook: configure GCC for: final/full build" - $src/configure \ - --libexecdir=/usr/lib \ - --enable-nls \ - --enable-languages=${languages} \ - --enable-shared \ + + mkdir build + cd build + SED=sed \ + ../configure \ + --enable-languages=$languages \ + --disable-multilib \ + --disable-bootstrap \ --with-system-zlib \ - --enable-clocale=gnu \ - --enable-objc-gc \ - --enable-__cxa_atexit \ - --enable-lto \ - --enable-threads=posix \ - --disable-bootstrap \ --with-pkgversion="$pkgversion" \ - ${CONFIGURE_ARGS} ${ARCH_ARGS} + $CONFIGURE_ARGS $ARCH_ARGS ;; esac && - make && make install + make && make install || return 1 + + ln -sv ../usr/bin/cpp $install/lib + ln -sv gcc $install/usr/bin/cc + install -v -dm755 $install/usr/lib/bfd-plugins + ln -sfv ../../libexec/gcc/$(gcc -dumpmachine)/$VERSION/liblto_plugin.so \ + $install/usr/lib/bfd-plugins/ + # ARM packages use another libs... case "$ARCH" in @@ -93,16 +105,38 @@ ;; esac - # Remove build directory. - rm -rf $WOK/$PACKAGE/source/$PACKAGE-build - rm -rf $WOK/$PACKAGE/$PACKAGE-build + + # Place the debugging symbols for selected libraries in separate files. + # This debugging information is needed if running regression tests that use valgrind or gdb. + # http://www.linuxfromscratch.org/lfs/view/stable/chapter06/strippingagain.html + + [ "$STAGE" == 'first' ] || + for i in libquadmath.so.0.0.0 libstdc++.so.6.0.24 libmpx.so.2.0.1 \ + libmpxwrappers.so.2.0.1 libitm.so.1.0.0 libcilkrts.so.5.0.0 \ + libatomic.so.1.2.0; do + cd $install/usr/lib/ + objcopy --only-keep-debug $i $i.dbg + strip --strip-unneeded $i + objcopy --add-gnu-debuglink=$i.dbg $i + done } genpkg_rules() { + case $STAGE in + first) + # single package 'gcc' on the first stage + copy @std @dev + DEPENDS="glibc-base gmp mpc-library mpfr zlib binutils elfutils" + CAT="development|first pass" + TAGS="LFS compiler C" + return 0 + ;; + esac + case $PACKAGE in gcc-lib-base) copy libgcc_s.so* libstdc++.so* - rm $fs/usr/lib/*.py + rm $fs/usr/lib*/*.py CAT="libs|base libraries" DEPENDS="glibc-base" ;; @@ -138,11 +172,11 @@ DEPENDS=" " ;; gcc) - copy bin/ lib/ include/ *.py @rm + copy @std @dev @rm DEPENDS="gcc-lib-base libgomp libobjc libatomic \ glibc-base gmp mpc-library mpfr zlib binutils elfutils" SUGGESTED="python" - TAGS="compiler C" + TAGS="LFS compiler C" ;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 gdbm/receipt --- a/gdbm/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/gdbm/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,23 +1,21 @@ # SliTaz package receipt v2. PACKAGE="gdbm" -VERSION="1.12" +VERSION="1.13" CATEGORY="misc" SHORT_DESC="GNU database manager" MAINTAINER="pascal.bellard@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/gdbm" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/gdbm.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" -BUILD_DEPENDS="gettext" +BUILD_DEPENDS="gettext readline-dev" SPLIT="gdbm-tools gdbm-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure \ --disable-static \ --enable-libgdbm-compat \ @@ -25,15 +23,15 @@ make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in gdbm) copy *.so* + TAGS="LFS" ;; gdbm-tools) copy bin/ + DEPENDS="gdbm ncurses readline" CAT="misc|tools" ;; *-dev) copy @dev;; diff -r c3c324c07d3f -r 90a5eb560fd6 gettext/receipt --- a/gettext/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/gettext/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,25 +7,24 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/gettext/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/gettext.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" +BUILD_DEPENDS_arm="attr-dev ncurses-dev libxml2-tools libxml2-dev" BUILD_DEPENDS="libcroco-dev libxml2-dev glib-dev acl-dev" -BUILD_DEPENDS_arm="attr-dev ncurses-dev libxml2-tools libxml2-dev" SPLIT="gettext-base gettext-tools gettext" -# Handle cross compilation. Glibc-locale are installed in cross chroot. -case "$ARCH" in - arm) ARCH_ARGS="--disable-acl" ;; -esac +compile_rules() { + case "$ARCH" in + arm) ARCH_ARGS="--disable-acl";; + esac -# Rules to configure and make the package. -compile_rules() -{ - ./configure $CONFIGURE_ARGS $ARCH_ARGS && - make && make install + fix ld + ./configure $CONFIGURE_ARGS $ARCH_ARGS && make && make install || return 1 + + chmod 0755 $install/usr/lib/preloadable_libintl.so # stripped gettext.sh rm $install/usr/bin/gettext.sh @@ -33,14 +32,11 @@ } # Just to be sure when cross compiling, gettext is part of base system. -testsuite() -{ +testsuite() { readelf -h $install/usr/bin/gettext } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in gettext-base) copy gettext gettext.sh envsubst ngettext @@ -51,13 +47,16 @@ copy msgcat msgfmt msgmerge xgettext \ libgettextlib*.so* libgettextsrc*.so* CAT="libs|tools" - DEPENDS="acl attr glib libcroco libgomp liblzma libxml2 pcre zlib" + DEPENDS="acl attr glib libcroco libgomp liblzma libxml2 ncurses \ + pcre zlib" ;; gettext) copy @std @dev @rm + rm -rf $fs/usr/share/doc/ # @dev files here # Runtime depends, then dev-depends DEPENDS="gettext-base gettext-tools libcroco liblzma libxml2 \ acl-dev attr-dev libcroco-dev libxml2-dev glib-dev pcre-dev m4" + TAGS="LFS" ;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 glibc/receipt --- a/glibc/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/glibc/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,18 +1,18 @@ # SliTaz package receipt v2. PACKAGE="glibc" -VERSION="2.25" +VERSION="2.26" CATEGORY="meta" SHORT_DESC="The GNU C libraries" MAINTAINER="pankso@slitaz.org" LICENSE="GPL2" WEB_SITE="http://www.gnu.org/software/libc/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/glibc.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" -BUILD_DEPENDS="linux-api-headers autoconf bash gawk gettext bison \ -texinfo" +BUILD_DEPENDS="linux-api-headers autoconf bash gawk gettext bison texinfo" # slitaz-i18n should present in wok SPLIT="glibc-base glibc-locale nscd glibc-dev" @@ -21,129 +21,142 @@ # Handle multiarch compilation. case "$ARCH" in - arm*|x86_64) VERSION="2.13";; + arm*) VERSION="2.13";; esac # Handle multiarch installation. case "$SLITAZ_ARCH" in - arm*|x86_64) VERSION="2.13";; + arm*) VERSION="2.13";; esac # Rules to compile & install the temporary toolchain. -cook_tmp_toolchain() -{ +cook_tmp_toolchain() { unset CFLAGS CXXFLAGS - echo "CFLAGS += -O2 -march=${ARCH/i386/i486} -mtune=${ARCH/i386/i486}" > configparms + case $ARCH in + i386) Arch='i486';; + x86_64) Arch='nocona';; + *) Arch="$ARCH";; + esac + echo "CFLAGS += -O2 -march=$Arch -mtune=$Arch" > configparms - { - # Build in a separate directory. - mkdir glibc-build - cd glibc-build + # Build in a separate directory. + mkdir build + cd build - ../configure \ - --host=$HOST_SYSTEM \ - --build=$($src/scripts/config.guess) \ - --disable-profile \ - --enable-add-ons \ - --enable-kernel=2.6.32 \ - --with-headers=/tools/include \ - libc_cv_forced_unwind=yes \ - libc_cv_c_cleanup=yes && - make -j1 && - make install - } || return 1 + ../configure \ + --host=$HOST_SYSTEM \ + --build=$($src/scripts/config.guess) \ + --disable-profile \ + --enable-add-ons \ + --enable-kernel=3.2 \ + --with-headers=/tools/include \ + libc_cv_forced_unwind=yes \ + libc_cv_c_cleanup=yes && + make -j1 && + make install || return 1 # Link compiler to this new glibc. SPECS=$(dirname $($HOST_SYSTEM-gcc -print-libgcc-file-name))/specs - $HOST_SYSTEM-gcc -dumpspecs | sed \ + $HOST_SYSTEM-gcc -dumpspecs \ + | sed \ -e 's@/lib\(64\)\?/ld@/tools&@g' \ -e "/^\*cpp:$/{n;s,$, -isystem /tools/include,}" > $SPECS unset SPECS } # Rules to configure and make the package. -compile_rules() -{ +compile_rules() { # Glibc needs ld.so.conf in the install destdir. mkdir -p $install/etc touch $install/etc/ld.so.conf # Read the INSTALL file in glibc. Also Glibc don't build with -Os flag. - # --enale-kernel use latest SliTaz Kernel version. From Glibc INSTALL: - # "The higher the VERSION number is, the less compatibility code is - # added, and the faster the code gets." unset CFLAGS CXXFLAGS - echo "CFLAGS += -O2 -march=${ARCH/i386/i486} -mtune=${ARCH/i386/i486}" > configparms + case $ARCH in + i386) Arch='i486';; + x86_64) Arch='nocona';; + *) Arch="$ARCH";; + esac + echo "CFLAGS += -O2 -march=$Arch -mtune=$Arch" > configparms - { - mkdir build - cd build - $src/configure \ - --disable-profile \ - --enable-add-ons \ - --enable-obsolete-rpc \ - --enable-kernel=2.6.32 \ - --libexecdir=/usr/lib/glibc \ - --build=$HOST_SYSTEM \ - --host=$HOST_SYSTEM \ - --target=$BUILD_SYSTEM && - make -j1 && make install_root=$DESTDIR install - } || return 1 + # Remove a file that may be left over from a previous build attempt + rm -f /usr/include/limits.h 2>/dev/null + + mkdir build + cd build + # --enale-kernel: please respect Kernel version of the build host, + # put the exact or lower numbers. From Glibc INSTALL: + # > The higher the VERSION number is, the less compatibility code is + # > added, and the faster the code gets. + CC="gcc -isystem /usr/lib/gcc/$BUILD_SYSTEM/$(. $WOK/gcc/receipt; echo $VERSION)/include \ + -isystem /usr/include" \ + ../configure \ + --disable-werror \ + --enable-kernel=3.2 \ + --enable-stack-protector=strong \ + libc_cv_slibdir=/lib \ + && + make -j1 && + touch $install/etc/ld.so.conf && + sed '/test-installation/s@$(PERL)@echo not running@' -i ../Makefile && + make install_root=$install install || return 1 # nscd - mkdir -p $install/etc/init.d $install/var/db - cp $stuff/nscd $install/etc/init.d - chmod +x $install/etc/init.d/nscd + cp ../nscd/nscd.conf $install/etc/nscd.conf + mkdir -p $install/var/cache/nscd $install/var/db + install -Dm755 $stuff/nscd $install/etc/init.d/nscd # nsswitch.conf cp $stuff/nsswitch.conf $install/etc + # ld.so.conf + install -Dm644 $stuff/ld.so.conf $install/etc/ld.so.conf + # If temporary toolchain was previously used, switch to regular toolchain. if [ -d /tools ]; then mv /tools/bin/ld /tools/bin/ld-old mv /tools/$(gcc -dumpmachine)/bin/ld /tools/$(gcc -dumpmachine)/bin/ld-old mv /tools/bin/ld-new /tools/bin/ld ln -s /tools/bin/ld /tools/$(gcc -dumpmachine)/bin/ld - gcc -dumpspecs | sed -e 's@/tools@@g' \ + gcc -dumpspecs \ + | sed -e 's@/tools@@g' \ -e '/\*startfile_prefix_spec:/{n;s@.*@/usr/lib/ @}' \ -e '/\*cpp:/{n;s@$@ -isystem /usr/include@}' > \ $(dirname $(gcc --print-libgcc-file-name))/specs fi -} -# i18n -get_locales() -{ - local path=$1 - for i in $DEFAULT_LOCALE; do - cp $install/$path/i18n/locales/$i $fs/usr/share/i18n/locales + # Create symlinks to the dynamic loader for LSB compliance. + case $ARCH in + i?86) + ln -sf ld-linux.so.2 $install/lib/ld-lsb.so.3 + ;; + x86_64) + mkdir -p $install/lib64 + ln -sf ../lib/ld-$VERSION.so $install/lib64/ld-linux-x86-64.so.2 + ln -sf ../lib/ld-$VERSION.so $install/lib64/ld-lsb-x86-64.so.3 + ;; + esac + + # Place the debugging symbols for selected libraries in separate files. + # This debugging information is needed if running regression tests that use valgrind or gdb. + # http://www.linuxfromscratch.org/lfs/view/stable/chapter06/strippingagain.html + + for i in ld-$VERSION.so libc-$VERSION.so libpthread-$VERSION.so libthread_db-1.0.so; do + cd $install/lib/ + objcopy --only-keep-debug $i $i.dbg + strip --strip-unneeded $i + objcopy --add-gnu-debuglink=$i.dbg $i done } -mk_dirs() -{ - mkdir -p \ - $fs/etc \ - $fs/lib \ - $fs/usr/bin \ - $fs/usr/lib/gconv \ - $fs/usr/lib/locale \ - $fs/usr/share/zoneinfo/America \ - $fs/usr/share/zoneinfo/Europe \ - $fs/usr/share/i18n/locales \ - $fs/usr/share/i18n/charmaps \ - $fs/usr/share/locale -} - -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in glibc) LOCALE="" CAT="meta|meta-package" DEPENDS="glibc-base glibc-locale glibc-dev" + TAGS="LFS" ;; glibc-base) case "$ARCH" in @@ -154,13 +167,6 @@ install="/cross/$ARCH/sysroot" copy libc_nonshared.a libpthread_nonshared.a ;; - x86_64) - # EXPERIMENTAL: Glibc is cross compiled by cross to have a - # toolchain so we can use these files instead of recooking it. - echo "Using cross compiled Glibc..." - install="/usr/cross/$ARCH" - copy libc_nonshared.a libpthread_nonshared.a - ;; esac copy \ ld-*.so* ld.so* \ @@ -192,20 +198,13 @@ nsswitch.conf \ $DEFAULT_LOCALE - case "$ARCH" in - x86_64) - # Fix libraries search path - sed -i "s|/usr/cross/$ARCH||g" $fs/lib/libc.so - sed -i "s|/usr/cross/$ARCH||g" $fs/lib/libpthread.so - ;; - esac + find $fs -type f -name '*.so.dbg' -delete + CAT="base-system|minimal libraries and UTF-8 support for SliTaz" DEPENDS=" " ;; glibc-locale) - copy gconv/ locale/ i18n/ iconv tzselect - # Remove glibc-base files - remove_already_packed + copy gconv/ locale/ i18n/ iconv tzselect @rm # Remove glibc-base files # Remove files provided by locale-pack. . $WOK/slitaz-i18n/stuff/locale-pack.conf @@ -219,13 +218,12 @@ DEPENDS="glibc-base" ;; nscd) - copy /var/db/ nscd getent nscd.conf + copy /var/db/ /var/cache/nscd/ nscd getent nscd.conf CAT="base-system|name-server caching daemon" DEPENDS=" " ;; glibc-dev) - copy @std @dev - remove_already_packed + copy @std @dev @rm DEPENDS="glibc-base" ;; esac diff -r c3c324c07d3f -r 90a5eb560fd6 glibc/stuff/ld.so.conf --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/glibc/stuff/ld.so.conf Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,4 @@ +# Begin /etc/ld.so.conf +/usr/local/lib +/opt/lib + diff -r c3c324c07d3f -r 90a5eb560fd6 glibc/stuff/patches/glibc-2.25-fhs-1.patch --- a/glibc/stuff/patches/glibc-2.25-fhs-1.patch Wed Feb 21 18:10:55 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,72 +0,0 @@ -Submitted By: Armin K. -Date: 2013-02-11 -Initial Package Version: 2.17 -Upstream Status: Not Applicable -Origin: Self -Description: This patch removes references to /var/db directory which is not part - of FHS and replaces them with more suitable directories in /var - hierarchy - /var/cache/nscd for nscd and /var/lib/nss_db for nss_db. - ---- a/Makeconfig 2012-12-25 04:02:13.000000000 +0100 -+++ b/Makeconfig 2013-02-11 01:32:32.500667439 +0100 -@@ -250,7 +250,7 @@ - - # Directory for the database files and Makefile for nss_db. - ifndef vardbdir --vardbdir = $(localstatedir)/db -+vardbdir = $(localstatedir)/lib/nss_db - endif - inst_vardbdir = $(install_root)$(vardbdir) - ---- a/nscd/nscd.h 2012-12-25 04:02:13.000000000 +0100 -+++ b/nscd/nscd.h 2013-02-11 01:32:32.500667439 +0100 -@@ -112,11 +112,11 @@ - - - /* Paths of the file for the persistent storage. */ --#define _PATH_NSCD_PASSWD_DB "/var/db/nscd/passwd" --#define _PATH_NSCD_GROUP_DB "/var/db/nscd/group" --#define _PATH_NSCD_HOSTS_DB "/var/db/nscd/hosts" --#define _PATH_NSCD_SERVICES_DB "/var/db/nscd/services" --#define _PATH_NSCD_NETGROUP_DB "/var/db/nscd/netgroup" -+#define _PATH_NSCD_PASSWD_DB "/var/cache/nscd/passwd" -+#define _PATH_NSCD_GROUP_DB "/var/cache/nscd/group" -+#define _PATH_NSCD_HOSTS_DB "/var/cache/nscd/hosts" -+#define _PATH_NSCD_SERVICES_DB "/var/cache/nscd/services" -+#define _PATH_NSCD_NETGROUP_DB "/var/cache/nscd/netgroup" - - /* Path used when not using persistent storage. */ - #define _PATH_NSCD_XYZ_DB_TMP "/var/run/nscd/dbXXXXXX" ---- a/nss/db-Makefile 2012-12-25 04:02:13.000000000 +0100 -+++ b/nss/db-Makefile 2013-02-11 01:32:32.500667439 +0100 -@@ -22,7 +22,7 @@ - /etc/rpc /etc/services /etc/shadow /etc/gshadow \ - /etc/netgroup) - --VAR_DB = /var/db -+VAR_DB = /var/lib/nss_db - - AWK = awk - MAKEDB = makedb --quiet ---- a/sysdeps/generic/paths.h 2012-12-25 04:02:13.000000000 +0100 -+++ b/sysdeps/generic/paths.h 2013-02-11 01:32:32.500667439 +0100 -@@ -68,7 +68,7 @@ - /* Provide trailing slash, since mostly used for building pathnames. */ - #define _PATH_DEV "/dev/" - #define _PATH_TMP "/tmp/" --#define _PATH_VARDB "/var/db/" -+#define _PATH_VARDB "/var/lib/nss_db/" - #define _PATH_VARRUN "/var/run/" - #define _PATH_VARTMP "/var/tmp/" - ---- a/sysdeps/unix/sysv/linux/paths.h 2012-12-25 04:02:13.000000000 +0100 -+++ b/sysdeps/unix/sysv/linux/paths.h 2013-02-11 01:32:32.504000831 +0100 -@@ -68,7 +68,7 @@ - /* Provide trailing slash, since mostly used for building pathnames. */ - #define _PATH_DEV "/dev/" - #define _PATH_TMP "/tmp/" --#define _PATH_VARDB "/var/db/" -+#define _PATH_VARDB "/var/lib/nss_db/" - #define _PATH_VARRUN "/var/run/" - #define _PATH_VARTMP "/var/tmp/" - diff -r c3c324c07d3f -r 90a5eb560fd6 glibc/stuff/patches/glibc-2.26-fhs-1.patch --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/glibc/stuff/patches/glibc-2.26-fhs-1.patch Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,72 @@ +Submitted By: Armin K. +Date: 2013-02-11 +Initial Package Version: 2.17 +Upstream Status: Not Applicable +Origin: Self +Description: This patch removes references to /var/db directory which is not part + of FHS and replaces them with more suitable directories in /var + hierarchy - /var/cache/nscd for nscd and /var/lib/nss_db for nss_db. + +--- a/Makeconfig 2012-12-25 04:02:13.000000000 +0100 ++++ b/Makeconfig 2013-02-11 01:32:32.500667439 +0100 +@@ -250,7 +250,7 @@ + + # Directory for the database files and Makefile for nss_db. + ifndef vardbdir +-vardbdir = $(localstatedir)/db ++vardbdir = $(localstatedir)/lib/nss_db + endif + inst_vardbdir = $(install_root)$(vardbdir) + +--- a/nscd/nscd.h 2012-12-25 04:02:13.000000000 +0100 ++++ b/nscd/nscd.h 2013-02-11 01:32:32.500667439 +0100 +@@ -112,11 +112,11 @@ + + + /* Paths of the file for the persistent storage. */ +-#define _PATH_NSCD_PASSWD_DB "/var/db/nscd/passwd" +-#define _PATH_NSCD_GROUP_DB "/var/db/nscd/group" +-#define _PATH_NSCD_HOSTS_DB "/var/db/nscd/hosts" +-#define _PATH_NSCD_SERVICES_DB "/var/db/nscd/services" +-#define _PATH_NSCD_NETGROUP_DB "/var/db/nscd/netgroup" ++#define _PATH_NSCD_PASSWD_DB "/var/cache/nscd/passwd" ++#define _PATH_NSCD_GROUP_DB "/var/cache/nscd/group" ++#define _PATH_NSCD_HOSTS_DB "/var/cache/nscd/hosts" ++#define _PATH_NSCD_SERVICES_DB "/var/cache/nscd/services" ++#define _PATH_NSCD_NETGROUP_DB "/var/cache/nscd/netgroup" + + /* Path used when not using persistent storage. */ + #define _PATH_NSCD_XYZ_DB_TMP "/var/run/nscd/dbXXXXXX" +--- a/nss/db-Makefile 2012-12-25 04:02:13.000000000 +0100 ++++ b/nss/db-Makefile 2013-02-11 01:32:32.500667439 +0100 +@@ -22,7 +22,7 @@ + /etc/rpc /etc/services /etc/shadow /etc/gshadow \ + /etc/netgroup) + +-VAR_DB = /var/db ++VAR_DB = /var/lib/nss_db + + AWK = awk + MAKEDB = makedb --quiet +--- a/sysdeps/generic/paths.h 2012-12-25 04:02:13.000000000 +0100 ++++ b/sysdeps/generic/paths.h 2013-02-11 01:32:32.500667439 +0100 +@@ -68,7 +68,7 @@ + /* Provide trailing slash, since mostly used for building pathnames. */ + #define _PATH_DEV "/dev/" + #define _PATH_TMP "/tmp/" +-#define _PATH_VARDB "/var/db/" ++#define _PATH_VARDB "/var/lib/nss_db/" + #define _PATH_VARRUN "/var/run/" + #define _PATH_VARTMP "/var/tmp/" + +--- a/sysdeps/unix/sysv/linux/paths.h 2012-12-25 04:02:13.000000000 +0100 ++++ b/sysdeps/unix/sysv/linux/paths.h 2013-02-11 01:32:32.504000831 +0100 +@@ -68,7 +68,7 @@ + /* Provide trailing slash, since mostly used for building pathnames. */ + #define _PATH_DEV "/dev/" + #define _PATH_TMP "/tmp/" +-#define _PATH_VARDB "/var/db/" ++#define _PATH_VARDB "/var/lib/nss_db/" + #define _PATH_VARRUN "/var/run/" + #define _PATH_VARTMP "/var/tmp/" + diff -r c3c324c07d3f -r 90a5eb560fd6 glibc/stuff/patches/glibc-po.patch --- a/glibc/stuff/patches/glibc-po.patch Wed Feb 21 18:10:55 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,249 +0,0 @@ -warning: internationalized messages should not contain the '\v' escape sequence - ---- a/po/cs.po -+++ b/po/cs.po -@@ -153,7 +153,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Generuje katalog zpráv.\vJe-li VSTUPNÍ_SOUBOR -, vstup bude načten ze standardního vstupu.\n" -+"Generuje katalog zpráv.\n" -+"Je-li VSTUPNÍ_SOUBOR -, vstup bude načten ze standardního vstupu.\n" - "Je-li VÝSTUPNÍ_SOUBOR -, výstup bude zapsán na standardní výstup.\n" - - #: catgets/gencat.c:123 ---- a/po/da.po -+++ b/po/da.po -@@ -131,7 +131,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Generér meddelelseskatalog.\vHvis INDFIL er '-' læses inddata fra standard ind.\n" -+"Generér meddelelseskatalog.\n" -+"Hvis INDFIL er '-' læses inddata fra standard ind.\n" - "Hvis UDFIL er '-' skrives uddata til standard ud.\n" - - #: catgets/gencat.c:123 ---- a/po/de.po -+++ b/po/de.po -@@ -131,7 +131,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Einen Message-Katalog generieren.\vWenn als INPUT-FILE - angegeben wird, wird\n" -+"Einen Message-Katalog generieren.\n" -+"Wenn als INPUT-FILE - angegeben wird, wird\n" - "von der Standardeingabe gelesen. Wenn die OUTPUT-FILE - ist, wird auf die\n" - "Standardausgabe geschrieben.\n" - ---- a/po/en_GB.po -+++ b/po/en_GB.po -@@ -1878,11 +1878,12 @@ - - #: catgets/gencat.c:116 - msgid "" --"Generate message catalog.\\vIf INPUT-FILE is -, input is read from standard " -+"Generate message catalog.\vIf INPUT-FILE is -, input is read from standard " - "input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Generate message catalogue.\\vIf INPUT-FILE is -, input is read from standard " -+"Generate message catalogue.\n" -+"If INPUT-FILE is -, input is read from standard " - "input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - ---- a/po/eo.po -+++ a/po/eo.po -@@ -138,7 +138,8 @@ - "is -, output is written to standard output.\n" - msgstr "" - " \n" --"Generas mesaĝan katalogon.\vSe ENIGDOSIERO estas «-», ĉefenigujo legiĝas. Se ELIGDOSIERO estas «-», la eligo skribiĝas al ĉefeligujo.\n" -+"Generas mesaĝan katalogon.\n" -+"Se ENIGDOSIERO estas «-», ĉefenigujo legiĝas. Se ELIGDOSIERO estas «-», la eligo skribiĝas al ĉefeligujo.\n" - - #: catgets/gencat.c:123 - msgid "" ---- a/po/es.po -+++ b/po/es.po -@@ -151,7 +151,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Genera un catálogo de mensajes.\vSi FICHERO-ENTRADA es -, la entrada\n" -+"Genera un catálogo de mensajes.\n" -+"Si FICHERO-ENTRADA es -, la entrada\n" - "se lee de la entrada estándar. Si FICHERO-SALIDA es -, el resultado se\n" - "escribe en la salida estándar.\n" - ---- a/po/fi.po -+++ b/po/fi.po -@@ -151,7 +151,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Luo viestikatalogi.\vJos SYÖTETIEDOSTO on -, syöte luetaan vakiosyötteestä. Jos TULOSTIEDOSTO\n" -+"Luo viestikatalogi.\n" -+"Jos SYÖTETIEDOSTO on -, syöte luetaan vakiosyötteestä. Jos TULOSTIEDOSTO\n" - "on -, tuloste kirjoitetaan vakiotulosteeseen.\n" - - #: catgets/gencat.c:123 ---- a/po/fr.po -+++ b/po/fr.po -@@ -136,7 +136,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Génère le catalogue de messages.\vSi le FICHIER_D_ENTRÉE est -, l'entrée est lue depuis l'entrée standard.\n" -+"Génère le catalogue de messages.\n" -+"Si le FICHIER_D_ENTRÉE est -, l'entrée est lue depuis l'entrée standard.\n" - "Si le FICHIER_DE_SORTIE est -, la sortie est dirigée sur la sortie standard.\n" - - #: catgets/gencat.c:123 ---- a/po/hr.po -+++ b/po/hr.po -@@ -132,7 +132,9 @@ - msgid "" - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" --msgstr "Napravi katalog poruka.\vAko je ULAZNA-DATOTEKA -, ulaz se čita sa standardnog ulaza. Ako je IZLAZNA-DATOTEKA -, izlaz se ispisuje na standardni izlaz.\n" -+msgstr "" -+"Napravi katalog poruka.\n" -+"Ako je ULAZNA-DATOTEKA -, ulaz se čita sa standardnog ulaza. Ako je IZLAZNA-DATOTEKA -, izlaz se ispisuje na standardni izlaz.\n" - - #: catgets/gencat.c:123 - msgid "" ---- a/po/hu.po -+++ b/po/hu.po -@@ -132,7 +132,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Üzenetkatalógus előállítása\v Ha a BEMENETIFÁJL a -, akkor a szabványos bemenetet olvassa. Ha a \n" -+"Üzenetkatalógus előállítása\n" -+"Ha a BEMENETIFÁJL a -, akkor a szabványos bemenetet olvassa. Ha a \n" - "KIMENETIFÁJL a -, akkor a szabványos kimenetre ír.\n" - - #: catgets/gencat.c:123 ---- a/po/ia.po -+++ b/po/ia.po -@@ -131,7 +131,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Generar un catalogo de messages.\vSi FILE-INPUT es -, le entrata se lege del input standard.\n" -+"Generar un catalogo de messages.\n" -+"Si FILE-INPUT es -, le entrata se lege del input standard.\n" - "Si FILE-OUTPUT es -, le resultato se scribe al output standard.\n" - - #: catgets/gencat.c:123 ---- a/po/it.po -+++ b/po/it.po -@@ -143,7 +143,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Genera un catalogo di messaggi.\vSe FILE-INPUT è -, l'input è letto dallo standard input.\n" -+"Genera un catalogo di messaggi.\n" -+"Se FILE-INPUT è -, l'input è letto dallo standard input.\n" - "Se FILE-OUTPUT è -, l'output è scritto sullo standard output.\n" - - #: catgets/gencat.c:123 ---- a/po/nl.po -+++ b/po/nl.po -@@ -145,7 +145,8 @@ - "is -, output is written to standard output.\n" - msgstr "" - " \n" --"Een berichtencatalogus genereren.\vAls INVOERBESTAND '-' is, wordt van standaardinvoer gelezen.\n" -+"Een berichtencatalogus genereren.\n" -+"Als INVOERBESTAND '-' is, wordt van standaardinvoer gelezen.\n" - "Als UITVOERBESTAND '-' is, wordt naar standaarduitvoer geschreven.\n" - - #: catgets/gencat.c:123 ---- a/po/pl.po -+++ b/po/pl.po -@@ -131,7 +131,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Generowanie katalogu komunikatów.\vJeśli PLIK-WE to -, dane są czytane ze standardowego wejścia. Jeśli PLIK-WY\n" -+"Generowanie katalogu komunikatów.\n" -+"Jeśli PLIK-WE to -, dane są czytane ze standardowego wejścia. Jeśli PLIK-WY\n" - "to -, dane są zapisywane na standardowe wyjście.\n" - - #: catgets/gencat.c:123 ---- a/po/ru.po -+++ b/po/ru.po -@@ -138,7 +138,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Генерирует каталог сообщений.\vЕсли ВХОДНОЙ-ФАЙЛ задан как -, читает стандартный ввод. Если\n" -+"Генерирует каталог сообщений.\n" -+"Если ВХОДНОЙ-ФАЙЛ задан как -, читает стандартный ввод. Если\n" - "ВЫХОДНОЙ-ФАЙЛ задан как -, результат печатается на стандартный вывод.\n" - - #: catgets/gencat.c:123 ---- a/po/sl.po -+++ b/po/sl.po -@@ -131,7 +131,9 @@ - msgid "" - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" --msgstr "Ustvari katalog sporočil.\vČe je VHODNA_DATOTEKA enaka -, se vhod bere s standardnega vhoda. Če je IZHODNA_DATOTEKA enaka -, se izhod zapisuje na standardni izhod.\n" -+msgstr "" -+"Ustvari katalog sporočil.\n" -+"Če je VHODNA_DATOTEKA enaka -, se vhod bere s standardnega vhoda. Če je IZHODNA_DATOTEKA enaka -, se izhod zapisuje na standardni izhod.\n" - - #: catgets/gencat.c:123 - msgid "" ---- a/po/sv.po -+++ b/po/sv.po -@@ -131,7 +131,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Skapa meddelandekatalog.\vOm INFIL är - så läses standard in. Om UTFIL\n" -+"Skapa meddelandekatalog.\n" -+"Om INFIL är - så läses standard in. Om UTFIL\n" - "är - så skrivs resultatet till standard ut.\n" - - #: catgets/gencat.c:123 ---- a/po/uk.po -+++ b/po/uk.po -@@ -133,7 +133,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"Створити каталог повідомлень.\vЯкщо як файл ВХІДНИЙ-ФАЙЛ\n" -+"Створити каталог повідомлень.\n" -+"Якщо як файл ВХІДНИЙ-ФАЙЛ\n" - "вказано -, вивід відбуватиметься у стандартний вивід.\n" - - #: catgets/gencat.c:123 ---- a/po/zh_CN.po -+++ b/po/zh_CN.po -@@ -137,7 +137,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"产生消息 catelog。\\v如果输入文件名是 -, 将从标准输入读取输入。\n" -+"产生消息 catelog。\n" -+"如果输入文件名是 -, 将从标准输入读取输入。\n" - "如果输出文件名是 -, 则输出将写到标准输出去。\n" - - #: catgets/gencat.c:123 ---- a/po/zh_TW.po -+++ b/po/zh_TW.po -@@ -134,7 +134,8 @@ - "Generate message catalog.\vIf INPUT-FILE is -, input is read from standard input. If OUTPUT-FILE\n" - "is -, output is written to standard output.\n" - msgstr "" --"產生訊息 catelog。\\v如果輸入檔名是 -, 將從標準輸入讀取輸入。\n" -+"產生訊息 catelog。\n" -+"如果輸入檔名是 -, 將從標準輸入讀取輸入。\n" - "如果輸出檔名是 -, 則輸出將寫到標準輸出去。\n" - - #: catgets/gencat.c:123 diff -r c3c324c07d3f -r 90a5eb560fd6 glibc/stuff/patches/series --- a/glibc/stuff/patches/series Wed Feb 21 18:10:55 2018 +0200 +++ b/glibc/stuff/patches/series Wed Feb 21 19:48:17 2018 +0200 @@ -1,2 +1,1 @@ -glibc-2.25-fhs-1.patch -glibc-po.patch +glibc-2.26-fhs-1.patch diff -r c3c324c07d3f -r 90a5eb560fd6 gmp/receipt --- a/gmp/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/gmp/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,7 +7,7 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="https://gmplib.org/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/gmp.html" TARBALL="$PACKAGE-$VERSION.tar.bz2" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" @@ -15,20 +15,20 @@ BUILD_DEPENDS="binutils m4 ncurses-dev readline-dev texinfo" SPLIT="gmp gmp-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { + # Create generic libraries + cp -v configfsf.guess config.guess + cp -v configfsf.sub config.sub + ./configure \ --enable-cxx \ --disable-static \ - --docdir=/usr/share/doc/gmp-$VERSION \ $CONFIGURE_ARGS && # -j > 1 make install fails. make && make html && make install && make install-html } -testsuite() -{ +testsuite() { checklog=$(mktemp) make check 2>&1 | tee $checklog pass=$(awk '/# PASS:/{total+=$3} ; END{print total}' $checklog) @@ -36,11 +36,12 @@ rm $checklog } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in - gmp) copy @std;; + gmp) + copy @std + TAGS="LFS" + ;; *-dev) copy @dev;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 gperf/receipt --- a/gperf/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/gperf/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,28 +1,22 @@ # SliTaz package receipt v2. PACKAGE="gperf" -VERSION="3.0.4" +VERSION="3.1" CATEGORY="development" SHORT_DESC="GNU gperf is a perfect hash function generator" MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/gperf/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/gperf.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" -# Rules to configure and make the package. -compile_rules() -{ - ./configure \ - --docdir=/usr/share/doc/gperf-$VERSION \ - $CONFIGURE_ARGS && - make && make install +compile_rules() { + ./configure $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 grep/receipt --- a/grep/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/grep/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,30 +1,28 @@ # SliTaz package receipt v2. PACKAGE="grep" -VERSION="3.0" +VERSION="3.1" CATEGORY="development" SHORT_DESC="GNU Global Regular Expression Print" MAINTAINER="paul@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/grep/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/grep.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" BUILD_DEPENDS="pcre-dev perl gettext" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure \ --bindir=/bin \ $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std DEPENDS="pcre" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 groff/receipt --- a/groff/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/groff/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,30 +1,28 @@ -# SliTaz package receipt. +# SliTaz package receipt v2. PACKAGE="groff" VERSION="1.22.3" CATEGORY="utilities" -SHORT_DESC="The GNU troff text-formatting system." +SHORT_DESC="The GNU troff text-formatting system" MAINTAINER="rocky@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/groff/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/groff.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" -BUILD_DEPENDS="ghostscript perl texinfo" +BUILD_DEPENDS="perl texinfo" # ghostscript -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { PAGE=A4 ./configure \ --without-x \ $CONFIGURE_ARGS && - PATH=$PATH:$src/src/preproc/preconv make && + PATH=$PATH:$src/src/preproc/preconv make -j1 && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy bin/ groff/ + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 grub2/receipt --- a/grub2/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/grub2/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,23 +1,22 @@ # SliTaz package receipt v2. PACKAGE="grub2" -SOURCE="grub" VERSION="2.02" CATEGORY="base-system" SHORT_DESC="GRUB2 boot loader" MAINTAINER="pascal.bellard@slitaz.org" LICENSE="GPL3" -TARBALL="$SOURCE-$VERSION.tar.xz" WEB_SITE="http://www.gnu.org/software/grub/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/grub.html" + +TARBALL="grub-$VERSION.tar.xz" WGET_URL="ftp://ftp.gnu.org/gnu/grub/$TARBALL" BUILD_DEPENDS="flex bison zlib-dev xz-dev freetype-dev ncurses-dev \ libusb-compat-dev libusb-dev libdevmapper-dev" -SPLIT="grub2 grub2-efi" +SPLIT="grub2 grub2-efi:efi" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { sed -i 's|sys/types.h>|&\n#include |' \ util/getroot.c grub-core/kern/emu/hostdisk.c @@ -25,64 +24,73 @@ sed -i "s/'.b'/& \\&\\& cur_len/" grub-core/lib/crypto.c grub-core/normal/auth.c # Fixes for flex 2.5.37 - export CFLAGS="-Wno-error=sign-compare -Wno-error=unused-value" + export CFLAGS="-Wno-error=sign-compare -Wno-error=unused-value" sed -i '/#undef gets/{N;s/.*/#ifdef gets\n&\n#endif/}' grub-core/gnulib/stdio*h sed -i 's/YY_FATAL_ERROR/REMOVED_&/' grub-core/script/yylex.l - - #chmod +x install-sh - ./configure --prefix=/usr --sysconfdir=/etc \ - --mandir=/usr/share/man $CONFIGURE_ARGS && - make $MAKEFLAGS && - make DESTDIR=$DESTDIR install - export EFI_ARCH=i386 - ./configure --prefix=/usr --sysconfdir=/etc \ - --with-platform=efi --target=${EFI_ARCH} --program-prefix="" \ - --mandir=/usr/share/man $CONFIGURE_ARGS && - make clean && - make $MAKEFLAGS - cd grub-core - ../grub-mkimage -d . -o ../bootia32.efi -O i386-efi -p /boot/grub \ - ntfs hfs appleldr boot cat efi_gop efi_uga elf fat hfsplus \ - iso9660 linux keylayouts memdisk minicmd part_apple ext2 extcmd \ - xfs xnu part_bsd part_gpt search search_fs_file chain btrfs \ - loadbios loadenv lvm minix minix2 reiserfs memrw mmap msdospart \ - scsi loopback normal configfile gzio all_video efi_gop efi_uga \ - gfxterm gettext echo boot chain -} + case $SET in + '') + #chmod +x install-sh + ./configure $CONFIGURE_ARGS && + make $MAKEFLAGS && + make DESTDIR=$DESTDIR install || return 1 -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ - case $PACKAGE in - grub2) - DEPENDS="zlib freetype ncurses libusb-compat grep libusb libdevmapper" - mkdir -p $fs/boot/grub $fs/usr - cp -a $install/usr/bin $fs/usr - cp -a $install/usr/sbin $fs/usr - cp -a $install/usr/share $fs/usr - cp -a $install/usr/lib $fs/usr - cp -a $install/etc $fs - - # Example config file (grub.cfg). - cp stuff/example-grub.cfg $fs/boot/grub - ;; - grub2-efi) - mkdir -p $fs/boot/efi/boot - cp $src/bootia32.efi $fs/boot/efi/boot - ;; + # Example config file (grub.cfg) + install -Dm0644 $stuff/example-grub.cfg $install/boot/grub/example-grub.cfg + ;; + efi) + export EFI_ARCH=i386 + ./configure \ + --with-platform=efi \ + --target=$EFI_ARCH \ + --program-prefix="" \ + $CONFIGURE_ARGS && + make clean && + make $MAKEFLAGS || return 1 + + cd grub-core + ../grub-mkimage \ + -d . \ + -o ../bootia32.efi \ + -O i386-efi \ + -p /boot/grub \ + ntfs hfs appleldr boot cat efi_gop efi_uga elf fat hfsplus \ + iso9660 linux keylayouts memdisk minicmd part_apple ext2 extcmd \ + xfs xnu part_bsd part_gpt search search_fs_file chain btrfs \ + loadbios loadenv lvm minix minix2 reiserfs memrw mmap msdospart \ + scsi loopback normal configfile gzio all_video efi_gop efi_uga \ + gfxterm gettext echo boot chain || return 1 + + install -Dm644 $src/bootia32.efi $install/boot/efi/boot/bootia32.efi + ;; esac } -post_install_grub2() -{ - cat < $install/usr/bin/menuconfig < $install/usr/bin/menuconfig chmod a+x $install/usr/bin/menuconfig - patch -p1 -i $stuff/installmandocs.patch - make mandocs && make installmandocs + if [ "$ARCH" == 'i386' ]; then + sed -i 's|uname -m|echo i386|; s|/bin/bash|/bin/ash|g' Makefile + make ARCH=um mrproper + cp $stuff/mini.config . + make ARCH=um allnoconfig KCONFIG_ALLCONFIG=mini.config && + make ARCH=um || return 1 - sed -i 's/uname -m/echo i386/;s|/bin/bash|/bin/ash|g' Makefile - make ARCH=um mrproper - cat > mini.config << EOF && -CONFIG_BINFMT_ELF=y -CONFIG_LBD=y -CONFIG_BLK_DEV=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_STDERR_CONSOLE=y -CONFIG_SSL=y -CONFIG_UNIX98_PTYS=y -CONFIG_PROC_FS=y -CONFIG_SYSFS_FS=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_UML_NET=y -CONFIG_UML_NET_TUNTAP=y -CONFIG_BLK_DEV_INITRD=y -CONFIG_PTY_CHAN=y -CONFIG_TTY_CHAN=y -CONFIG_NET=y -CONFIG_INET=y -CONFIG_UML_NET=y -CONFIG_UML_NET_TUNTAP=y -CONFIG_BLK_DEV_UBD=y -CONFIG_EXT4_FS=y -CONFIG_EXT4_USE_FOR_EXT23=y -CONFIG_EXT4_FS_XATTR=y -CONFIG_EXT4_FS_POSIX_ACL=y -CONFIG_FUSE_FS=y -CONFIG_ISO9660_FS=y -CONFIG_VFAT_FS=y -CONFIG_TMPFS=y -CONFIG_SWAP=y -CONFIG_SYSVIPC=y -CONFIG_SYSVIPC_SYSCTL=y -CONFIG_SYSVIPC_COMPAT=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_NETFILTER=y -CONFIG_NETFILTER_ADVANCED=y -CONFIG_NF_CONNTRACK=y -CONFIG_NETFILTER_XTABLES=y -CONFIG_NF_DEFRAG_IPV4=y -CONFIG_NF_CONNTRACK_IPV4=y -CONFIG_NF_CONNTRACK_PROC_COMPAT=y -CONFIG_IP_NF_IPTABLES=y -CONFIG_IP_NF_FILTER=y -CONFIG_IP_NF_TARGET_REJECT=y -CONFIG_NF_NAT=y -CONFIG_NF_NAT_NEEDED=y -CONFIG_IP_NF_TARGET_MASQUERADE=y -CONFIG_IP_NF_TARGET_REDIRECT=y -CONFIG_NETDEVICES=y -CONFIG_TUN=y -EOF - make ARCH=um allnoconfig KCONFIG_ALLCONFIG=mini.config && - make ARCH=um || return 1 - - mkdir -p $install/boot - cp $src/linux $install/boot/linux-uml-$VERSION - cat > $install/boot/vm-uml < $install/boot/vm-uml + chmod 755 $install/boot/vm-uml + fi } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in - linux-dev) - copy Kconfig* mconf menuconfig .config - CAT="development|menu based configuration tool" - ;; - linux-man) - copy /usr/share/man/ - CAT="doc|manual pages" - ;; - linux-api-headers) - copy @dev - CAT="development|headers sanitized for use in userspace" - PROVIDE="linux-headers linux64-api-headers linux64-headers" - ;; - linux-uml) - copy boot/ - ;; + linux-dev) + copy Kconfig* mconf menuconfig .config + CAT="development|menu based configuration tool" + DEPENDS="ncurses" + TAGS="LFS" + ;; + linux-api-headers) + copy @dev + DEPENDS=" " + CAT="development|headers sanitized for use in userspace" + PROVIDE="linux-headers linux64-api-headers linux64-headers" + ;; + linux-uml) + copy boot/ + CAT="development|UML" + DEPENDS=" " + ;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 linux-dev/stuff/menuconfig.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-dev/stuff/menuconfig.in Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,4 @@ +#!/bin/sh +p=$PWD +cd ../src/linux-@VERSION@-slitaz +SRCARCH=x86 ARCH=@ARCH@ KERNELVERSION=@VERSION@ $p/mconf Kconfig diff -r c3c324c07d3f -r 90a5eb560fd6 linux-dev/stuff/mini.config --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-dev/stuff/mini.config Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,51 @@ +CONFIG_BINFMT_ELF=y +CONFIG_LBD=y +CONFIG_BLK_DEV=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_STDERR_CONSOLE=y +CONFIG_SSL=y +CONFIG_UNIX98_PTYS=y +CONFIG_PROC_FS=y +CONFIG_SYSFS_FS=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_UML_NET=y +CONFIG_UML_NET_TUNTAP=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_PTY_CHAN=y +CONFIG_TTY_CHAN=y +CONFIG_NET=y +CONFIG_INET=y +CONFIG_UML_NET=y +CONFIG_UML_NET_TUNTAP=y +CONFIG_BLK_DEV_UBD=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_USE_FOR_EXT23=y +CONFIG_EXT4_FS_XATTR=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_FUSE_FS=y +CONFIG_ISO9660_FS=y +CONFIG_VFAT_FS=y +CONFIG_TMPFS=y +CONFIG_SWAP=y +CONFIG_SYSVIPC=y +CONFIG_SYSVIPC_SYSCTL=y +CONFIG_SYSVIPC_COMPAT=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_NETFILTER=y +CONFIG_NETFILTER_ADVANCED=y +CONFIG_NF_CONNTRACK=y +CONFIG_NETFILTER_XTABLES=y +CONFIG_NF_DEFRAG_IPV4=y +CONFIG_NF_CONNTRACK_IPV4=y +CONFIG_NF_CONNTRACK_PROC_COMPAT=y +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_NF_NAT=y +CONFIG_NF_NAT_NEEDED=y +CONFIG_IP_NF_TARGET_MASQUERADE=y +CONFIG_IP_NF_TARGET_REDIRECT=y +CONFIG_NETDEVICES=y +CONFIG_TUN=y diff -r c3c324c07d3f -r 90a5eb560fd6 linux-dev/stuff/vm-uml.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/linux-dev/stuff/vm-uml.in Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,17 @@ +#!/bin/sh + +# /etc/inittab: "tty21::respawn:/bin/su -c '/boot/vm-uml 1 /boot/guests/slitaz.img 2 512m' nobody" +n=${1:-0} +tap=tap$n +args="rw root=/dev/null" +dev="initrd=$(ls -r /boot/rootfs*.gz | sed q)" +if [ -n "$2" ]; then + args="ro root=620${3:-1} screen=text sound=noconf" + dev="ubd0=$2" +fi + +cd $(dirname $0) +ifconfig $tap 192.168.$n.1 +./linux-uml-@VERSION@ $dev mem=${4:-512m} fakehd fake_ide ubd=3 con0=fd:0,fd:1 \ + con=pts ssl=pts eth0=tuntap,$tap,fe:f0:00:00:00:0$n,192.168.$n.1 \ + $args cpuinfo=$(sed '/model name/!d;s/.*: //;s/ /_/g;q' /proc/cpuinfo) diff -r c3c324c07d3f -r 90a5eb560fd6 m4/receipt --- a/m4/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/m4/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,22 +7,17 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/m4/m4.html" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/m4.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" -DEPENDS="glibc-base" -BUILD_DEPENDS="" - -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ - copy m4 +genpkg_rules() { + copy @std + DEPENDS="glibc-base" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 make/receipt --- a/make/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/make/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,21 +7,18 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/make/" -HOST_ARCH="i486 arm x86_64" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/make.html" TARBALL="$PACKAGE-$VERSION.tar.bz2" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" BUILD_DEPENDS="gettext" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 man-db/receipt --- a/man-db/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/man-db/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,6 +7,7 @@ MAINTAINER="al.bobylev@gmail.com" LICENSE="GPL2" WEB_SITE="http://www.nongnu.org/man-db/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/man-db.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="http://download.savannah.gnu.org/releases/man-db/$TARBALL" @@ -15,11 +16,8 @@ zlib-dev" SPLIT="man-db-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure \ - --docdir=/usr/share/doc/$PACKAGE-$VERSION \ --sysconfdir=/etc \ --disable-setuid \ --with-browser=/usr/bin/lynx \ @@ -31,13 +29,12 @@ sed -i "s:man root:root root:g" $install/usr/lib/tmpfiles.d/man-db.conf } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in man-db) copy @std DEPENDS="gdbm libpipeline zlib" + TAGS="LFS" ;; *-dev) copy @dev;; esac diff -r c3c324c07d3f -r 90a5eb560fd6 man-pages/receipt --- a/man-pages/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/man-pages/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,24 +1,22 @@ # SliTaz package receipt v2. PACKAGE="man-pages" -VERSION="4.09" +VERSION="$(. $WOK/linux/receipt; echo $KBASEVER)" CATEGORY="doc" SHORT_DESC="Linux manual pages" MAINTAINER="al.bobylev@gmail.com" LICENSE="unknown" WEB_SITE="https://www.kernel.org/doc/man-pages/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/man-pages.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="https://www.kernel.org/pub/linux/docs/man-pages/$TARBALL" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { make DESTDIR=$install install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy man/ + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 mpc-library/receipt --- a/mpc-library/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/mpc-library/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,7 +7,7 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL2" WEB_SITE="http://www.multiprecision.org/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/mpc.html" TARBALL="mpc-$VERSION.tar.gz" WGET_URL="${WEB_SITE}mpc/download/$TARBALL" @@ -15,22 +15,19 @@ BUILD_DEPENDS="mpfr-dev gmp-dev texinfo" SPLIT="mpc-library-dev" -compile_rules() -{ +compile_rules() { ./configure \ --disable-static \ - --docdir=/usr/share/doc/mpc-$VERSION \ $CONFIGURE_ARGS && make && make html && make install && make install-html } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in mpc-library) copy @std DEPENDS="mpfr gmp" + TAGS="LFS" ;; *-dev) copy @dev diff -r c3c324c07d3f -r 90a5eb560fd6 mpfr/receipt --- a/mpfr/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/mpfr/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,7 +7,7 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="http://www.mpfr.org/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/mpfr.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="http://www.mpfr.org/$PACKAGE-$VERSION/$TARBALL" @@ -15,30 +15,31 @@ BUILD_DEPENDS="gmp-dev texinfo" SPLIT="mpfr-dev" +compile_rules() { #--with-gmp-lib=/cross/$ARCH/sysroot/usr/lib #--with-gmp-include=/cross/$ARCH/sysroot/usr/include -# Rules to configure and make the package. -compile_rules() -{ ./configure \ --disable-static \ --enable-thread-safe \ - --docdir=/usr/share/doc/mpfr-$VERSION \ $CONFIGURE_ARGS && make && make html && make install && make install-html } -testsuite() -{ - case "$ARCH" in i?86) make check;; esac +testsuite() { + case "$ARCH" in + arm*) ;; + *) make check;; + esac } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in - mpfr) copy @std; DEPENDS="gmp";; + mpfr) + copy @std + DEPENDS="gmp" + TAGS="LFS" + ;; *-dev) copy @dev;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 ncurses/receipt --- a/ncurses/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/ncurses/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,19 +7,17 @@ MAINTAINER="pankso@slitaz.org" LICENSE="MIT" WEB_SITE="https://www.gnu.org/software/ncurses/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/ncurses.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="http://ftp.gnu.org/gnu/ncurses/$TARBALL" -HOST_ARCH="i486 arm" +BUILD_DEPENDS_arm=" " BUILD_DEPENDS="flex" -BUILD_DEPENDS_arm="" SPLIT="ncurses-libform ncurses-libmenu ncurses-libpanel ncurses-extra \ ncurses-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { sed -i '/LIBTOOL_INSTALL/d' c++/Makefile.in ./configure \ @@ -61,14 +59,13 @@ ln -s xterm-256color $install/usr/share/terminfo/x/xterm } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in ncurses) copy clear tset reset libncursesw.so* tabset/ \ linux xterm xterm-256color # ansi rxvt vt100 vt102* xterm-color xterm-new xterm-vt220 + TAGS="LFS" ;; ncurses-libform) copy libformw.so* @@ -86,8 +83,7 @@ DEPENDS="ncurses" ;; ncurses-extra) - copy @std - remove_already_packed + copy @std @rm CAT="base-system|extra files" DEPENDS="ncurses" ;; diff -r c3c324c07d3f -r 90a5eb560fd6 patch/receipt --- a/patch/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/patch/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,33 +7,19 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="http://savannah.gnu.org/projects/patch/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/patch.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" BUILD_DEPENDS="attr-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ - copy patch +genpkg_rules() { + copy @std DEPENDS="attr" + TAGS="LFS" } - -## Prevent erasing busybox... -#pre_install() -#{ -# rm -f "$1/usr/bin/patch" -#} -# -#post_remove() -#{ -# ln -s /bin/busybox "$1/usr/bin/patch" -#} diff -r c3c324c07d3f -r 90a5eb560fd6 perl-xml-parser/receipt --- a/perl-xml-parser/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/perl-xml-parser/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,25 +7,22 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL" WEB_SITE="https://metacpan.org/release/XML-Parser" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/xml-parser.html" TARBALL="XML-Parser-$VERSION.tar.gz" WGET_URL="https://cpan.metacpan.org/authors/id/T/TO/TODDR/$TARBALL" -DEPENDS="perl-libwww-perl expat" -BUILD_DEPENDS="$DEPENDS perl-dev expat-dev" +BUILD_DEPENDS="perl-libwww-perl expat perl-dev expat-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { perl Makefile.PL && make && make DESTDIR=$install install && chmod -R u+w $install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std + DEPENDS="perl-libwww-perl expat" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 perl/receipt --- a/perl/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/perl/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,68 +1,91 @@ # SliTaz package receipt v2. PACKAGE="perl" -VERSION="5.24.1" +VERSION="5.26.1" CATEGORY="development" SHORT_DESC="Perl interpreter and modules" MAINTAINER="pankso@slitaz.org" LICENSE="GPL" WEB_SITE="https://www.perl.org/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/perl.html" TARBALL="$PACKAGE-$VERSION.tar.bz2" WGET_URL="http://www.cpan.org/src/5.0/$TARBALL" COOKOPTS="!perlz" BUILD_DEPENDS="zlib-dev bzip2-dev less gdbm-dev" -SPLIT="perl-core perl-dev" -SIBLINGS="microperl perl-thread" +#SPLIT="perl-core perl perl-dev microperl:micro" +SPLIT="perl-core perl perl-dev" +# microperl: waiting for 5.26.2: https://rt.perl.org/Public/Bug/Display.html?id=132255 +# sv.c:(.text+0x1aa95): undefined reference to `Perl_fp_class_denorm' # Rules to compile & install the temporary toolchain. -cook_tmp_toolchain() -{ +cook_tmp_toolchain() { cd $src - { sh Configure -des -Dprefix=/tools \ + sh Configure -des \ + -Dprefix=/tools \ -Dstatic_ext='Data/Dumper Fcntl IO' && - + # Only few tools are needed in the tmp toolchain. - make perl utilities ext/Errno/pm_to_blib - } || return 1 + make perl utilities ext/Errno/pm_to_blib || return 1 + cp perl pod/pod2man /tools/bin mkdir -p /tools/lib/perl5/$VERSION cp -R lib/* /tools/lib/perl5/$VERSION } -# Rules to configure and make the package. -compile_rules() -{ - export BUILD_ZLIB=False - export BUILD_BZIP2=0 - sh Configure -des \ - -Dprefix=/usr \ - -Dvendorprefix=/usr \ - -Dman1dir=/usr/share/man/man1 \ - -Dman3dir=/usr/share/man/man3 \ - -Dpager="/usr/bin/less -isR" \ - -Duseshrplib && - make && make install +compile_rules() { + case $SET in + micro) + patch -p1 -i $stuff/microperl.patch + sed -i.orig "s|usr/local|usr|; + s|perl5/${VERSION%.*}|perl5/$VERSION|; + s|unknown|$HOST_SYSTEM|" uconfig.sh uconfig64.sh + + case $ARCH in + x86_64) make -f Makefile.micro regen_uconfig64;; + *) make -f Makefile.micro regen_uconfig;; + esac && + make -f Makefile.micro && + install -Dm755 microperl $install/usr/bin/microperl + ;; + *) + export BUILD_ZLIB=False + export BUILD_BZIP2=0 + sh Configure -des \ + -Dprefix=/usr \ + -Dvendorprefix=/usr \ + -Dman1dir=/usr/share/man/man1 \ + -Dman3dir=/usr/share/man/man3 \ + -Dpager="/usr/bin/less -isR" \ + -Duseshrplib \ + -Dusethreads && + make && make install + ;; + esac } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in - perl) - copy @std - find $fs \( -name perl -o -name perl$VERSION -o -name libperl.so \) -delete - DEPENDS="bzlib gdbm zlib perl-core" - PROVIDE="microperl" - ;; perl-core) - # perl binary depends on libperl.so copy perl perl$VERSION libperl.so DEPENDS="glibc-base" CAT="development|minimal interpreter" ;; - *-dev) copy @dev;; + perl) + copy @std @rm + DEPENDS="bzlib gdbm zlib perl-core" + PROVIDE="microperl perl-thread" + TAGS="LFS" + ;; + *-dev) + copy @dev + ;; + microperl) + copy @std + CAT="development|micro version" + DEPENDS="glibc-base" + ;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 perl/stuff/microperl.patch --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/perl/stuff/microperl.patch Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,40 @@ +--- a/Makefile.micro ++++ b/Makefile.micro +@@ -1,7 +1,7 @@ + LD = $(CC) + CCFLAGS = -c + DEFINES = -DPERL_CORE -DPERL_MICRO -DSTANDARD_C -DPERL_USE_SAFE_PUTENV \ +- -DNO_MATHOMS ++ -DNO_MATHOMS -DUSE_PERLIO + OPTIMIZE = + CFLAGS = $(DEFINES) $(OPTIMIZE) + LDFLAGS = +@@ -14,7 +14,7 @@ + + all: microperl + +-O = uav$(_O) udeb$(_O) udoio$(_O) udoop$(_O) udquote$(_O) udump$(_O) \ ++O = uav$(_O) ucaretx$(_O) udeb$(_O) udoio$(_O) udoop$(_O) udquote$(_O) udump$(_O) \ + uglobals$(_O) ugv$(_O) uhv$(_O) umro$(_O)\ + umg$(_O) uperlmain$(_O) uop$(_O) ureentr$(_O) \ + upad$(_O) uperl$(_O) uperlio$(_O) uperly$(_O) upp$(_O) \ +@@ -66,6 +66,9 @@ + + uav$(_O): $(HE) av.c + $(CC) $(CCFLAGS) -o $@ $(CFLAGS) av.c ++ ++ucaretx$(_O): $(HE) caretx.c ++ $(CC) $(CCFLAGS) -o $@ $(CFLAGS) caretx.c + + udeb$(_O): $(HE) deb.c + $(CC) $(CCFLAGS) -o $@ $(CFLAGS) deb.c +--- a/EXTERN.h ++++ b/EXTERN.h +@@ -57,3 +57,7 @@ + #define INIT(x) + + #undef DOINIT ++ ++#define I_LIMITS ++#define I_FCNTL ++#define I_FLOAT diff -r c3c324c07d3f -r 90a5eb560fd6 perl/stuff/miniperlmain.patch --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/perl/stuff/miniperlmain.patch Wed Feb 21 19:48:17 2018 +0200 @@ -0,0 +1,16 @@ +--- miniperlmain.c.orig ++++ miniperlmain.c +@@ -117,13 +117,6 @@ + if (!exitstatus) + perl_run(my_perl); + +- /* Unregister our signal handler before destroying my_perl */ +- for (i = 0; PL_sig_name[i]; i++) { +- if (rsignal_state(PL_sig_num[i]) == (Sighandler_t) PL_csighandlerp) { +- rsignal(PL_sig_num[i], (Sighandler_t) SIG_DFL); +- } +- } +- + exitstatus = perl_destruct(my_perl); + + perl_free(my_perl); diff -r c3c324c07d3f -r 90a5eb560fd6 pkg-config/receipt --- a/pkg-config/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/pkg-config/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,23 +1,21 @@ # SliTaz package receipt v2. PACKAGE="pkg-config" -VERSION="0.29.1" +VERSION="0.29.2" CATEGORY="development" SHORT_DESC="Free desktop packages manager" MAINTAINER="pankso@slitaz.org" LICENSE="GPL2" WEB_SITE="https://www.freedesktop.org/wiki/Software/pkg-config/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/pkg-config.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="https://pkg-config.freedesktop.org/releases/$TARBALL" +BUILD_DEPENDS_arm=" " BUILD_DEPENDS="gcc libtool perl" -BUILD_DEPENDS_arm=" " -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { case "$ARCH" in arm) ./configure \ @@ -27,13 +25,9 @@ --cache-file=arm-linux.cache ;; *) ./configure \ - --prefix=/usr \ --with-internal-glib \ - --disable-compile-warnings \ --disable-host-tool \ - --docdir=/usr/share/doc/pkg-config-$VERSION \ - --build=$HOST_SYSTEM \ - --host=$HOST_SYSTEM + $CONFIGURE_ARGS ;; esac && make && make install @@ -43,9 +37,8 @@ ln -s ${TOOLPREFIX}pkg-config pkg-config } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std @dev DEPENDS="glibc-base" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 procps-ng/receipt --- a/procps-ng/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/procps-ng/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,6 +7,7 @@ MAINTAINER="al.bobylev@gmail.com" LICENSE="GPL2 LGPL2" WEB_SITE="https://gitlab.com/procps-ng/procps" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/procps-ng.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$SF_MIRROR/$PACKAGE/$TARBALL" @@ -14,17 +15,15 @@ BUILD_DEPENDS="gettext ncurses-dev" SPLIT="procps-ng-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { +# --docdir=/usr/share/doc/procps-ng-$VERSION \ ./configure \ --exec-prefix= \ --libdir=/usr/lib \ - --docdir=/usr/share/doc/procps-ng-$VERSION \ --disable-static \ --disable-kill \ $CONFIGURE_ARGS && - make && make install + make && make install || return 1 mkdir -p $install/lib mv $install/usr/lib/libprocps.so.* $install/lib @@ -32,13 +31,12 @@ $install/usr/lib/libprocps.so } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in procps-ng) copy @std DEPENDS="gettext-base ncurses" + TAGS="LFS" ;; *-dev) copy @dev;; esac diff -r c3c324c07d3f -r 90a5eb560fd6 psmisc/receipt --- a/psmisc/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/psmisc/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,21 +1,20 @@ # SliTaz package receipt v2. PACKAGE="psmisc" -VERSION="22.21" +VERSION="23.1" CATEGORY="system-tools" SHORT_DESC="PSmisc - Small utilities that use the /proc filesystem" MAINTAINER="al.bobylev@gmail.com" LICENSE="GPL2" WEB_SITE="http://psmisc.sourceforge.net/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/psmisc.html" -TARBALL="$PACKAGE-$VERSION.tar.gz" +TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$SF_MIRROR/psmisc/$TARBALL" BUILD_DEPENDS="gettext ncurses-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure $CONFIGURE_ARGS && make && make install mkdir -p $install/bin @@ -23,9 +22,8 @@ mv $install/usr/bin/killall $install/bin } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std DEPENDS="ncurses" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 readline/receipt --- a/readline/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/readline/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -6,8 +6,8 @@ SHORT_DESC="GNU readline" MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" -WEB_SITE="http://cnswww.cns.cwru.edu/php/chet/readline/rltop.html" -HOST_ARCH="i486 arm" +WEB_SITE="https://www.gnu.org/software/readline" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/readline.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" @@ -15,9 +15,7 @@ BUILD_DEPENDS="ncurses-dev flex" SPLIT="readline-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { case "$ARCH" in arm*) export bash_cv_wcwidth_broken=true ;; esac @@ -27,30 +25,31 @@ ./configure \ --disable-static \ - --libdir=/lib \ - --docdir=/usr/share/doc/readline-$VERSION \ $CONFIGURE_ARGS && - make SHLIB_LIBS=-lncurses && - make DESTDIR=$install install + make SHLIB_LIBS="-lncursesw" && + make SHLIB_LIBS="-lncurses" DESTDIR=$install install || return 1 - # *.so.* -> /lib - # *.so -> /usr/lib - mkdir -p $install/usr/lib - ln -sfv ../../lib/$(readlink $install/lib/libreadline.so) \ - $install/usr/lib/libreadline.so - ln -sfv ../../lib/$(readlink $install/lib/libhistory.so ) \ - $install/usr/lib/libhistory.so - rm $install/lib/*.so + mkdir -p $install/lib + for i in readline history; do + mv -v $install/usr/lib/lib$i.so.* $install/lib + ln -sf ../../lib/$(readlink $install/usr/lib/lib$i.so) \ + $install/usr/lib/lib$i.so + done + + find $install -type f -name '*.so*' -exec chmod 755 '{}' \; + rmdir --ignore-fail-on-non-empty $install/usr/bin/ # install the documentation - cp doc/*.pdf doc/*.html $install/usr/share/doc/readline-7.0 + cook_pick_docs doc/*.pdf doc/*.html } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in - readline) copy @std; DEPENDS="ncurses";; + readline) + copy @std + DEPENDS="ncurses" + TAGS="LFS" + ;; *-dev) copy @dev;; esac } diff -r c3c324c07d3f -r 90a5eb560fd6 sed/receipt --- a/sed/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/sed/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,29 +7,27 @@ MAINTAINER="paul@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/sed/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/sed.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" BUILD_DEPENDS="acl-dev gettext texinfo" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { # Rebuild after patching msgfmt po/ru.po -o po/ru.gmo ./configure \ --bindir=/bin \ - --htmldir=/usr/share/doc/sed-$VERSION \ $CONFIGURE_ARGS && - make $MAKEFLAGS && make html && make install && make -C doc install-html + make $MAKEFLAGS && make html && make install || return 1 + + cook_pick_docs doc/sed.html } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std DEPENDS="acl" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 shadow/receipt --- a/shadow/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/shadow/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,16 +7,21 @@ MAINTAINER="al.bobylev@gmail.com" LICENSE="BSD" WEB_SITE="http://pkg-shadow.alioth.debian.org/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/shadow.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="https://github.com/shadow-maint/shadow/releases/download/$VERSION/$TARBALL" -DEPENDS="acl attr" BUILD_DEPENDS="acl-dev attr-dev gettext" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { + # Disable the installation of the `groups` program and its man pages, as + # Coreutils provides a better version. + sed -i 's/groups$(EXEEXT) //' src/Makefile.in + find man -name Makefile.in -exec sed -i 's/groups\.1 / /' {} \; + find man -name Makefile.in -exec sed -i 's/getspnam\.3 / /' {} \; + find man -name Makefile.in -exec sed -i 's/passwd\.5 / /' {} \; + sed -i -e 's|#ENCRYPT_METHOD DES|ENCRYPT_METHOD SHA512|' \ -e 's|/var/spool/mail|/var/mail|' etc/login.defs sed -i 's|bash|sh|' etc/useradd @@ -29,8 +34,8 @@ mv $install/usr/bin/passwd $install/bin } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std + DEPENDS="acl attr" + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 sysklogd/receipt --- a/sysklogd/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/sysklogd/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,13 +7,12 @@ MAINTAINER="al.bobylev@gmail.com" LICENSE="GPL2" WEB_SITE="http://www.infodrom.org/projects/sysklogd/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/sysklogd.html" TARBALL="$PACKAGE-$VERSION.tar.gz" WGET_URL="${WEB_SITE}download/$TARBALL" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { sed -i '/Error loading kernel symbols/{n;n;d}' ksym_mod.c sed -i 's/union wait/int/' syslogd.c mkdir -p $install/sbin $install/usr/share/man/man5 $install/usr/share/man/man8 @@ -24,8 +23,7 @@ cp $stuff/syslog.conf $install/etc } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std + TAGS="LFS" } diff -r c3c324c07d3f -r 90a5eb560fd6 tar/receipt --- a/tar/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/tar/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,16 +7,14 @@ MAINTAINER="pankso@slitaz.org" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/tar/" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/tar.html" TARBALL="$PACKAGE-$VERSION.tar.bz2" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" BUILD_DEPENDS="acl-dev gettext texinfo" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { FORCE_UNSAFE_CONFIGURE=1 \ ./configure \ --bindir=/bin \ @@ -25,10 +23,8 @@ make -C doc install-html docdir=/usr/share/doc/tar-$VERSION } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std DEPENDS="acl attr" - TAGS="archive compression" + TAGS="LFS archive compression" } diff -r c3c324c07d3f -r 90a5eb560fd6 texinfo/receipt --- a/texinfo/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/texinfo/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,31 +1,28 @@ # SliTaz package receipt v2. PACKAGE="texinfo" -VERSION="6.3" +VERSION="6.4" CATEGORY="development" SHORT_DESC="GNU documentation tools" MAINTAINER="rcx@zoominternet.net" LICENSE="GPL3" WEB_SITE="https://www.gnu.org/software/texinfo/" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/texinfo.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$GNU_MIRROR/$PACKAGE/$TARBALL" BUILD_DEPENDS="perl-dev ncurses-dev gettext" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure \ --disable-static \ $CONFIGURE_ARGS && make && make install } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { copy @std @dev DEPENDS="perl" - TAGS="documentation" + TAGS="LFS documentation" } diff -r c3c324c07d3f -r 90a5eb560fd6 util-linux/receipt --- a/util-linux/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/util-linux/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -1,13 +1,13 @@ # SliTaz package receipt v2. PACKAGE="util-linux" -VERSION="2.30.0" +VERSION="2.30.1" CATEGORY="meta" SHORT_DESC="Random collection of Linux utilities" MAINTAINER="pankso@slitaz.org" LICENSE="GPL2 LGPL2.1 BSD PublicDomain" WEB_SITE="https://en.wikipedia.org/wiki/Util-linux" -HOST_ARCH="i486 arm" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/util-linux.html" TARBALL="$PACKAGE-${VERSION%.0}.tar.xz" WGET_URL="https://www.kernel.org/pub/linux/utils/util-linux/v${VERSION%.*}/$TARBALL" @@ -23,9 +23,7 @@ util-linux-sfdisk util-linux-smartcols util-linux-smartcols-dev \ util-linux-uuid util-linux-uuid-dev util-linux-whereis util-linux-misc" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { # Fix undeclared 'LINE_MAX' case "$ARCH" in arm) sed -i s'|LINE_MAX|25|' text-utils/*.c ;; @@ -44,9 +42,7 @@ cp $stuff/cfdisk.desktop $install/usr/share/applications } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case ${PACKAGE#util-linux-} in bash-completion) copy bash-completion/ @@ -182,10 +178,12 @@ DEPENDS="glibc-base" ;; misc) - copy @std @dev - remove_already_packed + copy @std @dev @rm CAT="system-tools|misc utilities" ;; + util-linux) + TAGS="LFS" + ;; esac if [ "$PACKAGE" != 'util-linux-bash-completion' -a -d "$fs/usr/share/bash-completion" ]; then diff -r c3c324c07d3f -r 90a5eb560fd6 xz/receipt --- a/xz/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/xz/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,8 +7,7 @@ MAINTAINER="pascal.bellard@slitaz.org" LICENSE="PublicDomain LGPL2.1 GPL2 GPL3" WEB_SITE="https://tukaani.org/xz/" -TAGS="compression archive" -HOST_ARCH="i486 arm x86_64" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/xz.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$WEB_SITE$TARBALL" @@ -16,15 +15,12 @@ BUILD_DEPENDS="gettext" SPLIT="xz-tools xz liblzma xz-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure \ --disable-static \ - --docdir=/usr/share/doc/xz-$VERSION \ --enable-small \ $CONFIGURE_ARGS && - make && make install + make && make install || return 1 mkdir -p $install/bin $install/lib for i in lzma unlzma lzcat xz unxz xzcat; do @@ -35,9 +31,7 @@ $install/usr/lib/liblzma.so } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in xz-tools) copy /usr/bin/ @@ -45,9 +39,9 @@ DEPENDS="liblzma" ;; xz) - copy bin/ - remove_already_packed + copy bin/ @rm DEPENDS="liblzma" + TAGS="LFS compression archive" ;; liblzma) copy liblzma.so* diff -r c3c324c07d3f -r 90a5eb560fd6 zlib/receipt --- a/zlib/receipt Wed Feb 21 18:10:55 2018 +0200 +++ b/zlib/receipt Wed Feb 21 19:48:17 2018 +0200 @@ -7,19 +7,17 @@ MAINTAINER="pankso@slitaz.org" LICENSE="zlib/libpng" WEB_SITE="http://www.zlib.net/" -HOST_ARCH="i486 arm x86_64" +LFS="http://www.linuxfromscratch.org/lfs/view/stable/chapter06/zlib.html" TARBALL="$PACKAGE-$VERSION.tar.xz" WGET_URL="$WEB_SITE$TARBALL" SPLIT="zlib-dev" -# Rules to configure and make the package. -compile_rules() -{ +compile_rules() { ./configure \ --prefix=/usr && - make && make install + make && make install || return 1 mkdir -p $install/lib mv -v $install/usr/lib/libz.so.* $install/lib @@ -28,17 +26,13 @@ } # Important cross compiled package so run readelf. -testsuite() -{ - cd $install - readelf -h lib/*.so.$VERSION +testsuite() { + readelf -h $install/lib/libz.so.$VERSION } -# Rules to gen a SliTaz package suitable for Tazpkg. -genpkg_rules() -{ +genpkg_rules() { case $PACKAGE in - zlib) copy @std ;; - zlib-dev) copy @dev ;; + zlib) copy @std; TAGS="LFS";; + *-dev) copy @dev;; esac }