#!/bin/bash
#Single-Machine OpenHPC installation
ncpus=`cat /proc/cpuinfo | grep MH | wc -l`
HOST=`hostname`
#===BEGIN===setRepos

################################dnf -y update
#===END===setRepos

#===BEGIN===basicTools
yum -y install lrzsz lm_sensors
#yum -y install lrzsz ipmitool lm_sensors
#===END===basicTools

#===BEGIN===basicConfigs
# Set network auto connect
sed -i "s/^ONBOOT=.*$/ONBOOT=yes/" /etc/sysconfig/network-scripts/ifcfg-*

# Disable SELinux and Firewalld
echo "Disabling firewall and SELINUX.."
systemctl disable firewalld
systemctl stop firewalld
sed -i 's/^SELINUX=.*$/SELINUX=disabled/' /etc/selinux/config
setenforce 0

# save systemd-journal permanently
mkdir -p /var/log/journal
chown root:systemd-journal /var/log/journal
chmod 2775 /var/log/journal
systemctl restart systemd-journald.service
#===END===basicConfigs

#===BEGIN===OHPC-repoInstall
echo "OpenHPC Local repo"
yum install -y http://repos.openhpc.community/OpenHPC/2/EL_8/x86_64/ohpc-release-2-1.el8.x86_64.rpm
yum install -y dnf-plugins-core
yum config-manager --set-enabled powertools
yum -y install ohpc-base
yum -y install epel-release
yum -y install  lmod-ohpc
echo "export LMOD_AUTO_SWAP=yes" > /etc/profile.d/lmod_auto_swap.sh
echo 'export MODULEPATH=/opt/pub/modulefiles/toolkits:/opt/pub/modulefiles/softwares:/opt/pub/modulefiles/libs:$MODULEPATH' >> /etc/profile.d/lmod.sh
#===END===OHPC-repoInstall
#===BEGIN===towerSlurm
echo "Install slurm server"
### mariadb
dnf install -y mariadb mariadb-server
systemctl start mariadb
#rm -rf /var/lib/mysql/*
#systemctl restart mariadb
mysql -e "create user 'slurm'@'localhost' identified by 'password'; grant all on slurm_acct_db.* TO 'slurm'@'localhost'; show variables like 'have_innodb'; flush privileges; create database slurm_acct_db;"

cat >  /etc/my.cnf.d/innodb.cnf << EOF
[mysqld]
innodb_buffer_pool_size=1024M
innodb_log_file_size=64M
innodb_lock_wait_timeout=900
EOF

systemctl stop mariadb
mv /var/lib/mysql/ib_logfile? /tmp/

yum install -y ohpc-slurm-server
yum -y install ohpc-slurm-client
yum -y install nhc-ohpc
mkdir /var/log/slurm
chown slurm:slurm /var/log/slurm
cat > /etc/slurm/slurm.conf << EOF
ClusterName=local
ControlMachine=$HOST

SlurmUser=slurm
SlurmctldPort=6817
SlurmdPort=6818
AuthType=auth/munge
StateSaveLocation=/var/spool/slurm/ctld
SlurmdSpoolDir=/var/spool/slurm/d
SwitchType=switch/none
MpiDefault=none
SlurmctldPidFile=/var/run/slurmctld.pid
SlurmdPidFile=/var/run/slurmd.pid
ProctrackType=proctrack/pgid

# TIMERS
SlurmctldTimeout=300
SlurmdTimeout=300
InactiveLimit=0
MinJobAge=300
KillWait=30
Waittime=0

# SCHEDULING
SchedulerType=sched/backfill
SelectType=select/cons_tres
SelectTypeParameters=CR_Core

# LOGGING
SlurmctldDebug=info
SlurmctldLogFile=/var/log/slurm/slurmctld.log
SlurmdDebug=info
SlurmdLogFile=/var/log/slurm/slurmd.log
JobCompType=jobcomp/none

# ACCOUNTING
#JobAcctGatherType=jobacct_gather/linux
#JobAcctGatherFrequency=30

AccountingStorageType=accounting_storage/slurmdbd
AccountingStorageEnforce=limits

TaskPlugin=task/affinity
PropagateResourceLimitsExcept=MEMLOCK
#Epilog=/etc/slurm/slurm.epilog.clean
SlurmctldParameters=enable_configless
ReturnToService=2
HealthCheckProgram=/usr/sbin/nhc
HealthCheckInterval=300
# COMPUTE NODES
NodeName=$HOST CPUs=$ncpus State=UNKNOWN
PartitionName=compute Nodes=$HOST Default=YES MaxTime=UNLIMITED State=UP
EOF

cat > /etc/slurm/slurmdbd.conf << EOF
AuthType=auth/munge
DbdAddr=localhost
DbdHost=localhost
SlurmUser=slurm
DebugLevel=verbose
LogFile=/var/log/slurm/slurmdbd.log
PidFile=/var/run/slurmdbd.pid
StorageType=accounting_storage/mysql
StorageHost=localhost
StoragePass=password
StorageUser=slurm
StorageLoc=slurm_acct_db
EOF

chown slurm: /etc/slurm/slurmdbd.conf
chmod 600 /etc/slurm/slurmdbd.conf
chown slurm: /etc/slurm/slurm.conf
chmod 644 /etc/slurm/slurm.conf
mkdir -p /var/spool/slurm && chown -R slurm:slurm /var/spool/slurm
#===END===towerSlurm

#===BEGIN===OHPC-basics
echo "unlock memlock & stack"
echo "* soft memlock unlimited" > /etc/security/limits.conf
echo "* hard memlock unlimited" >> /etc/security/limits.conf
echo "* soft stack unlimited" >> /etc/security/limits.conf
echo "* hard stack unlimited" >> /etc/security/limits.conf

# Enable Intel oneAPI and install OpenHPC compatibility packages
yum -y install intel-oneapi-toolkit-release-ohpc
rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
yum -y install intel-compilers-devel-ohpc
yum -y install intel-mpi-devel-ohpc
# Optionally, choose the Omni-Path enabled build for MVAPICH2. Otherwise, skip to retain IB variant
yum -y install mvapich2-psm2-intel-ohpc
# Install 3rd party libraries/tools meta-packages built with Intel toolchain
yum -y install ohpc-intel-serial-libs
yum -y install ohpc-intel-geopm
yum -y install ohpc-intel-io-libs
yum -y install ohpc-intel-perf-tools
yum -y install ohpc-intel-python3-libs
yum -y install ohpc-intel-mpich-parallel-libs
yum -y install ohpc-intel-mvapich2-parallel-libs
yum -y install ohpc-intel-openmpi4-parallel-libs
yum -y install ohpc-intel-impi-parallel-libs
echo "install ohpc dev tools"
yum -y install ohpc-autotools EasyBuild-ohpc hwloc-ohpc spack-ohpc valgrind-ohpc gnu12-compilers-ohpc openmpi4-gnu12-ohpc mpich-ofi-gnu12-ohpc ohpc-gnu12-perf-tools
yum -y install ohpc-gnu12-serial-libs
yum -y install ohpc-gnu12-io-libs
yum -y install ohpc-gnu12-python-libs
yum -y install ohpc-gnu12-runtimes
yum -y install ohpc-gnu12-openmpi4-parallel-libs
yum -y install ohpc-gnu12-mpich-parallel-libs
yum -y install slurm-torque-ohpc
yum -y install perl-Switch
#===END===OHPC-basics
#===BEGIN===Set slurm mail server
yum -y install postfix mailx
systemctl enable postfix.service
systemctl start postfix.service
sed -i '/SlurmUser=slurm/i MailProg=\/usr\/bin\/smail' /etc/slurm/slurm.conf
echo 'set from=yanlicloud@163.com' >> /etc/mail.rc
echo 'set smtp=smtp.163.com' >> /etc/mail.rc
echo 'set smtp-auth=login' >> /etc/mail.rc
echo 'set smtp-auth-user=yanlicloud@163.com' >> /etc/mail.rc
echo 'set smtp-auth-password=PWGKGCCTQXLBYICO' >> /etc/mail.rc
echo 'set ssl-verify=ignore' >> /etc/mail.rc
#===END===Set slurm mail server
#===BEGIN===Slurmd
yum -y install ohpc-base-compute
yum -y install ohpc-slurm-client
systemctl enable munge
systemctl enable mariadb
systemctl enable slurmdbd
systemctl enable slurmctld
systemctl enable slurmd
systemctl start munge
systemctl start mariadb
systemctl start slurmdbd
sleep 2
systemctl start slurmctld
systemctl start slurmd
sleep 2
systemctl restart chronyd.service
systemctl enable chronyd.service
chronyc -c makestep
systemctl restart slurmctld.service
systemctl status slurmdbd.service
systemctl status slurmctld.service
sacctmgr -i create account local
#===END===Slurmd
#===BEGIN===Adduser
cat > /usr/bin/adduser-yanli <<EOF
#!/bin/bash
useradd \$1 -m
echo "\$1:\$2" | chpasswd

sudo -u \$1 ssh-keygen -b 2048 -t rsa -f /home/\$1/.ssh/id_rsa -q -N ""
cd /home/\$1/.ssh
cat id_rsa.pub >> authorized_keys
chmod 700 /home/\$1/.ssh
chmod 600 /home/\$1/.ssh/authorized_keys
chown -R \$1: /home/\$1/.ssh

sacctmgr add user \$1 account=local
EOF
