Building a Linux Cluster out of Windows Desktops

Using htcondor, windows7, vmware, and linux you can build a linux cluster on top of windows desktops.

Step 1
Install Htcondor, VMWare Player,  VIX, and VMWare Perl scripting API on Windows 7 machines.

use this as installer.bat to preload the msi installer with your pool data.  This requires you have a file at windows.php which loads a default config or overrides if it hostname.local is in the same dir.

REM CHTC URL Config HTCondor Windows7 Installer for CSL 
@echo on
cd /d "%~dp0"
set ARGS=
set ARGS=%ARGS% NEWPOOL=N
set ARGS=%ARGS% POOLNAME="CS"
REM RUNJOBS=c to use keyboard daemon to stop jobs when there is activity
set ARGS=%ARGS% RUNJOBS=C
set ARGS=%ARGS% VACATEJOBS=Y
REM SUBMITJOBS=N windows nodes no the cs pool are not submitters
set ARGS=%ARGS% SUBMITJOBS=N
set ARGS=%ARGS% CONDOREMAIL=""
REM defaults incase http get of real config fails
set ARGS=%ARGS% HOSTALLOWREAD="*.wisc.edu  execute-side@matchsession"
set ARGS=%ARGS% HOSTALLOWWRITE="*.cs.wisc.edu *.chtc.wisc.edu"
set ARGS=%ARGS% HOSTALLOWADMINISTATOR="condor.cs.wisc.edu"
set ARGS=%ARGS% INSTALLDIR="C:\Condor"
set ARGS=%ARGS% POOLHOSTNAME="condor.cs.wisc.edu"
set ARGS=%ARGS% ACCOUNTINGDOMAIN="none"
set ARGS=%ARGS% JVMLOCATION="java.exe"
set ARGS=%ARGS% SMTPSERVER=""
set ARGS=%ARGS% USEVMUNIVERSE=Y
set ARGS=%ARGS% VMNETWORKING=C
set ARGS=%ARGS% VMMAXNUMBER=1
set ARGS=%ARGS% VMMEMORY="$(DETECTED_MEMORY)/2"
set ARGS=%ARGS% PERLLOCATION="C:\sperl-5.18.2.1\perl.exe"
set ARGS=%ARGS% ACCOUNTINGDOMAIN=cs.wisc.edu
set ARGS=%ARGS% LOCAL_CONFIG_FILE = condor_urlfetch -TOOL 
http://url/windows.php?host=$(HOSTNAME) 
 /etc/condor/condor_configt.url_cache |
del c:\condor\condor_config
start /wait msiexec /q /l* "C:\Program Files\CSL\logs\condor-8.2.5.txt" /i condor-8.2.5-284806-Windows-x86.msi %ARGS% REBOOT=Suppress
net start condor 

# cat windows.php
<?php

$base_config = "WINDOWS7";
$base_config_lines = file($base_config);

$host = $_GET["host"] . ".local";
if (file_exists($host)) {
  $config_lines = file($host);
  $config = array_merge($base_config_lines, $config_lines);
}
else {
  $config = $base_config_lines;
}

foreach ($config as $line_num => $line ) {
  echo $line;
}

?>

# cat WINDOWS7
COLLECTOR_HOST = $(CONDOR_HOST)
CCB_ADDRESS = $(CONDOR_HOST)
PRIVATE_NETWORK_NAME = cs.wisc.edu
STARTER_ALLOW_RUNAS_OWNER = False
DYNAMIC_RUN_ACCOUNT_LOCAL_GROUP = Condor Reuse Users
ALLOW_WRITE = $(IP_ADDRESS)


#VM Policy
VM_GAHP_LOG = C:\Condor\log\VMGAHPLog
VMWARE_PERL=C:\sperl-5.18.2.1\perl\bin\perl.exe
VM_TYPE = vmware
VM_NETWORKING = TRUE
VM_NETWORKING_TYPE = NAT
VM_MEMORY= $(DETECTED_MEMORY) - 512
VM_MAX_NUMBER = $(DETECTED_CORES)

#preemption policy
use policy : desktop
#start jobs if the non condor load is under .3 and the keyboard has been idle for at least 60 min
START = (( LoadAvg - CondorLoadAvg) <= 0.3 ) && (KeyboardIdle > 60 * 60)
PREEMPT = ((Activity == "Suspended") && ((time() - EnteredCurrentActivity) > 10 * 60)) || (SUSPEND && (WANT_SUSPEND ==False))
KILL = $(SUSPEND)
#KILL = ((KeyboardIdle < 60) || ( (CpuBusyTime > 120) && ifThenElse(JobStart =!= UNDEFINED, (time() - JobStart),0 > 90)))
SLOTS_CONNECTED_TO_CONSOLE = $(NUM_CPUS)
SLOTS_CONNECTED_TO_KEYBOARD = $(SLOTS_CONNECTED_TO_CONSOLE)

#slot policy
NUM_SLOTS=1
NUM_SLOTS_TYPE_1=1
SLOT_TYPE_1_PARTITIONABLE=true
SLOT_TYPE_1 = cpus=100%,disk=100%,swap=100%,gpus=100%
ASSIGN_CPU_AFFINITY=true
use FEATURE : GPUs

#ADMIN Policy
CONDOR_ADMIN = admin@domain.com
HOSTALLOW_ADMINISTRATOR = $(CONDOR_HOST)


Step 2
Create VMWare VM which starts htcondor on boot and randomizes its name, I take a random entry from cracklib-dicts.

Step 3
Submit vm directory to your windows machines, and watch as the linux vms start up and take jobs that otherwise would not run on windows.

Using vmsl65 as the directory of your vmware image this submit file will send it out to windows hosts to create your linux vms:

#cat vmsl65.sub
universe = vm
executable = linux.iso
vm_type = vmware
vm_networking = true
vm_networking_type=nat
vmware_dir = vmsl65
vmware_should_transfer_files = YES
vmware_snapshot_disk = false
vm_memory = 2000
request_memory = 2004
request_disk = 20000000
log = ~/vmsl65_$(Cluster)_$(Process).log
notification = NEVER
requirements = Target.OpSys == "WINDOWS"
queue 5000

#condor_submit vmsl65.sub

References:
Thanks TJ!

https://twiki.grid.iu.edu/bin/view/CampusGrids/LinuxCondorPoolOnWindowsLabMachines