====== ISER/RMDA with ZFS on Ubuntu Server 17.04/17.10 ======


===== RDMA Infiniband Modules =====
<file bash /etc/modules>
rdma_cm
ib_uverbs
rdma_ucm
ib_ucm
ib_umad
ib_ipoib

mlx4_ib
mlx4_en
mlx5_ib
iw_cxgb3
iw_cxgb4
iw_nes
iw_c2
rpcrdma
</file>

<file bash /etc/security/limits.conf>
...
* soft memlock unlimited
* hard memlock unlimited
root soft memlock unlimited
root hard memlock unlimited
...
</file>

<file bash /etc/network/interfaces>
auto ib0
iface ib0 inet static
        address 192.168.113.1
        netmask 255.255.255.0

</file>
==== Comment out discovery command and add a new ISER Command ====
<file perl /usr/share/perl5/PVE/Storage/ISCSIPlugin.pm>
#my $cmd = [$ISCSIADM, '--mode', 'discovery', '--type', 'sendtargets', '--portal', $portal];
my $cmd = [$ISCSIADM, '--mode', 'discovery', '--type', 'sendtargets', '--portal', $portal, '-I', 'iser'];
</file>



==== URLS ====
  * [[https://community.mellanox.com/docs/DOC-1468|HowTo Configure LIO enabled with ISER for Ubuntu 14.04 Inbox driver]]
  * [[https://community.mellanox.com/docs/DOC-1465|HowTo Setup RDMA Connection using Inbox Driver (RHEL, Ubuntu)]]






===== Configure TargetCLI =====
==== Disable ISCSI Authentication ====
  * Remember on new version of Debian and Ubuntu **targetcli** has been replaced with **targetcli-fb** 
<file>
cd iscsi/iqn.*/tpgt1/ 
set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1
</file>
==== Enable ISCSI iSER ====
<file>
cd iscsi/iqn.*/tpgt1/portals/192.168.221.1:3260
enable_iser true
</file>

==== Create File based iSCSI backstore ====
<file>
cd /backstores/fileio/
create file_or_dev=/tank/vol1/iservol2 name=iservol2 size=20G write_back=true
</file>

==== Assign a new lun to a target ====
<file>
cd /iscsi/iqn.*/tpg1/luns/
create /backstores/fileio/iservol2
</file>

===== ISCSI Admin Commands =====
==== Discover Targets from ISCSI Portal (TCP) ====
<file>
iscsiadm -m discovery -p <IP>:<PORT>
</file>

==== Discover Targets from ISCSI Portal using iSER ====
<file>
iscsiadm -m discovery -p <IP>:<PORT> -I iser
</file>
==== Refresh LUNS of ISCSI Target ====
<file>
iscsiadm -m node -R
</file>
==== Logout of all Targets ====
<file>
iscsiadm -m node -u
</file>
==== Logout to all Targets ====
<file>
iscsiadm -m node -l
</file>

==== Logout of a Target ====
<file>
iscsiadm -m node -T <TARGETNAME> -u
</file>
==== Logout to a Target ====
<file>
iscsiadm -m node -T <TARGETNAME> -l
</file>

===== Other Commands =====
==== Check IO Delay/Sats ====
<file>
apt install sysstat
</file>
<file>
iostat -x 1
</file>
==== Check ZFS IO Stats ====
<file>
zpool iostat -v 1
</file>

==== FIO Performance Test ====
<file>
fio --rw=read --bs=128k --numjobs=1 --iodepth=3 --size="200m" --loops=1 --ioengine=libaio --direct=1 --invalidate=1 --fsync_on_close=1 --randrepeat=1 --norandommap --exitall --name "test1" --filename="/dev/sdf" --output=/tmp/fio_verify.sh-22334.log --time_based --runtime=305
</file>