LINUX.ORG.RU
решено ФорумAdmin

Что не так с pigz в этом скрипте?

 ,


0

1
#!/bin/bash -x

ListOfBases='dblist'
PORT=5433
USERNAME='admin'
SERVERNAME='localhost'
ARCHIVEDIR='/incrimental'
MAXDBCOUNT="1"
TEMPDIR='/cache'
RAMDISKSIZE="6144M"
SUFFIX=''
NETDIR='//192.168.0.15/incrimental'
ERRLOG='/var/log/archive.log'
STARTFILE=''
SERIALNUM=''
CHATID='-327777777'
BOTID='bot14807773775:AAH1y147777MaP_NyXjjjjjzB6uuuSJM'

if [[ 'daily' = $1 ]]
then
    SUFFIX='daily_'$(date +%Y-%m-%d-%H)
fi

if [[ 'monthly' = $1 ]]
then
    SUFFIX='monthly_'$(date +%Y-%m-%d)
fi

if [[ 'hot' = $1 ]]
then
    SUFFIX='hot_'$(date +%Y-%m-%d-%H%M)
fi

if [[ 'weekly' = $1 ]]
then
    SUFFIX='weekly_'$(date +%Y-%m-%d)
fi

if [[ $SUFFIX = '' ]]
then
    echo 'Type monthly, weekly, daily or hot as parametr'
    exit
fi


if mount | grep -qw $NETDIR
then
    echo "Network directory is mounted"
else
    echo "Network direcrory $NETDIR is not mounted" >> $ERRLOG
    MESSAGE="FAIL | pg_dump | PostgreSQL | ${SUFFIX}"
    /usr/bin/curl -s -X POST -H 'Content-Type: application/json' -d '{"chat_id": "'"$CHATID"'", "text": "'"$MESSAGE"'", "disable_notification": false}' https://api.telegram.org/$BOTID/sendMessage
    exit
fi

DBLIST=$(psql -U $USERNAME -p $PORT -l | q -d'|' "select c1 from - where c1 <> '' and c2 <> '' and c1 not like 'template%' and c1 not like '%_bak%' limit 1,$MAXDBCOUNT")

echo "$DBLIST" > ${ARCHIVEDIR}'/'$ListOfBases

mkdir -p "$TEMPDIR"
mount -t tmpfs -o "size=$RAMDISKSIZE" tmpfs "$TEMPDIR"
mkdir ${TEMPDIR}"/temp"

cat $ARCHIVEDIR'/'$ListOfBases | while read DBNAME
do
    mkdir -p ${ARCHIVEDIR}'/'${DBNAME}

    if [[ 'weekly' = $1 ]]
    then
        > ${ARCHIVEDIR}/${DBNAME}'/start'
        STARTFILE=''
    fi

    if ! [[ -e ${ARCHIVEDIR}/${DBNAME}'/start' ]]
    then
        > ${ARCHIVEDIR}/${DBNAME}'/start'
    fi

    STARTFILE="$(cat ${ARCHIVEDIR}/${DBNAME}'/start')"

    if [[ $STARTFILE == '' ]]
    then
        SERIALNUM=$(/usr/bin/pwgen 15 1)
        STARTFILE="${DBNAME}_${SERIALNUM}_${SUFFIX}.dump"
        echo "$STARTFILE" > ${ARCHIVEDIR}'/'${DBNAME}'/start'
        nice -n 19 ionice -c3 pg_dump -d $DBNAME -h ${SERVERNAME} -p $PORT -U ${USERNAME} -w > ${TEMPDIR}/temp/${STARTFILE}
        if [[ $? -ne 0 ]]
        then
            echo "$DBNAME: pg_dump error code is "$? >> $ERRLOG
        fi
        /usr/bin/rdiff signature ${TEMPDIR}/temp/${STARTFILE} ${ARCHIVEDIR}/${DBNAME}/${STARTFILE}.signature
        if [[ $? -ne 0 ]]
        then
            echo "$DBNAME: signature creation error "$? >> $ERRLOG
        fi
        /usr/bin/pigz ${TEMPDIR}/temp/${STARTFILE} > ${ARCHIVEDIR}/${DBNAME}/${STARTFILE}.gz
        if [[ $? -ne 0 ]]
        then
            echo "$DBNAME: start file archiving error "$? >> $ERRLOG
        fi
        rm -f ${TEMPDIR}/temp/${STARTFILE}
    else
        if [ 'hot' = $1 ] || [ 'monthly' = $1 ]
        then
            nice -n 19 ionice -c3 pg_dump -d $DBNAME -h ${SERVERNAME} -p $PORT -U ${USERNAME} -w | pigz > ${ARCHIVEDIR}'/'${DBNAME}'/'${DBNAME}'_'$SUFFIX.dump.gz;
            if [[ $? -ne 0 ]]
            then
                echo "$DBNAME: pg_dump error code is "$? >> $ERRLOG
            fi
        else
            nice -n 19 ionice -c3 pg_dump -d $DBNAME -h ${SERVERNAME} -p $PORT -U ${USERNAME} -w > ${TEMPDIR}/temp/${DBNAME}_$SUFFIX.dump
            if [[ $? -ne 0 ]]
            then
                echo "$DBNAME: pg_dump error code is "$? >> $ERRLOG
            fi
            SERIALNUM=$(echo "$STARTFILE" | sed 's/^.*${DBNAME}_//;s/_weekly.*//')
            /usr/bin/rdiff delta ${ARCHIVEDIR}/${DBNAME}/$STARTFILE.signature ${TEMPDIR}/temp/${DBNAME}_$SUFFIX.dump  ${ARCHIVEDIR}/${DBNAME}/${DBNAME}_${SERIALNUM}_${SUFFIX}.dump.delta
            if [[ $? -ne 0 ]]
            then
                echo "$DBNAME: delta creation error "$? >> $ERRLOG
            fi
            rm ${TEMPDIR}/temp/${DBNAME}_${SUFFIX}.dump
        fi
    fi
done

umount -f "$TEMPDIR"

BackupErr=$(stat $ERRLOG -c %s)

MESSAGE=''

if [[ $BackupErr = 0 ]]
then
    MESSAGE="OK | pg_dump | PostgreSQL | ${SUFFIX}"
else
    MESSAGE="FAIL | pg_dump | PostgreSQL | ${SUFFIX}"
fi

/usr/bin/curl -s -X POST -H 'Content-Type: application/json' -d '{"chat_id": "'"$CHATID"'", "text": "'"$MESSAGE"'", "disable_notification": false}' https://api.telegram.org/$BOTID/sendMessage

В общем pigz жмет архив в этом фрагменте, по времени видно что он работает:

        SERIALNUM=$(/usr/bin/pwgen 15 1)
        STARTFILE="${DBNAME}_${SERIALNUM}_${SUFFIX}.dump"
        echo "$STARTFILE" > ${ARCHIVEDIR}'/'${DBNAME}'/start'
        nice -n 19 ionice -c3 pg_dump -d $DBNAME -h ${SERVERNAME} -p $PORT -U ${USERNAME} -w > ${TEMPDIR}/temp/${STARTFILE}
        if [[ $? -ne 0 ]]
        then
            echo "$DBNAME: pg_dump error code is "$? >> $ERRLOG
        fi
        /usr/bin/rdiff signature ${TEMPDIR}/temp/${STARTFILE} ${ARCHIVEDIR}/${DBNAME}/${STARTFILE}.signature
        if [[ $? -ne 0 ]]
        then
            echo "$DBNAME: signature creation error "$? >> $ERRLOG
        fi
        /usr/bin/pigz ${TEMPDIR}/temp/${STARTFILE} > ${ARCHIVEDIR}/${DBNAME}/${STARTFILE}.gz
        if [[ $? -ne 0 ]]
        then
            echo "$DBNAME: start file archiving error "$? >> $ERRLOG
        fi
        rm -f ${TEMPDIR}/temp/${STARTFILE}

Но результирующий файл пустой то есть вести 0 байт



Последнее исправление: Shulman (всего исправлений: 1)

И ещё имеет смысл заменить pigz на zstd.
zstd тоже умеет работать в несколько потоков, но работает быстрее при тех же уровнях сжатия или сжимает лучше за то же время. Зависит от выбранных настроек.

i-rinat ★★★★★
()
Вы не можете добавлять комментарии в эту тему. Тема перемещена в архив.