How to Install XGBoost on Windows


When you googled and landed to this blog, I could say you are struggling to install XGBoost on your Python. Anyway, My environment is Windows 7 and I use Spyder Python 3.4.4. I like this Spyder, it has so many modules available such as Panda, Numpy, Seaborn, Matplotlib, etc but no XGBoost. Let’s me give you some tips on how to install it:

  1. Go to this GitHub, https://github.com/dmlc/xgboost and download the folder python-package. Save it into your python environment. …\python-package\xgboost. it depends on where you have your python.exe. I have several python versions on my computer.
  2. Go to this site, http://www.picnet.com.au/blogs/guido/post/2016/09/22/xgboost-windows-x64-binaries-for-download/ and download  the latest libxgboost.dll. And save it to the same location as number 1.
  3. run python setup.py install.
  4. Once finished, open python console and type import xgboost to check if your installation ran successfully or not.

 

Good Luck!!!!!

Collect All Service Properties Information in Site


You can find your service properties information from ArcGIS server manager or admin but how about if you want to see all of your services’ properties? you can run a script and see from there. I found this script was really useful when i wanted to check if any publisher published their service without accepting default value, for example: change minimum instances or maximum instances or maximum idle time or different cluster. Or with a little modification, you can query some other information.

Hopefully you find this script is useful for you.

Cheers…

 

#-------------------------------------------------------------------------------
# Name: List_Svc_properties.py
# Purpose: List All Started Services and their properties
#
# This script could be scheduled to run at a regular interval.
#
# Author: Muryadi Oey
#
# Created: 03/11/2016
# Copyright: (c) muryadioey 2016
# Licence: <your licence>
#-------------------------------------------------------------------------------

# For Http calls
import httplib, urllib,json,urllib2

# For system tools
import sys, datetime

# For reading passwords without echoing
import getpass

#Get the current date
now = datetime.datetime.now()

# Defines the entry point into the script
def main(argv=None):

# ##########################################################################################################################
# Start of Variables
# ##########################################################################################################################

# Server name
##serverName = raw_input("Enter server name: ")
serverName = 'SERVER01'

# Admin/publisher user name and password
## username = raw_input("Enter user name: ")
## password = getpass.getpass("Enter password: ")
username = 'ARCGISADMIN'
password = 'passwordadmin'

# Log file location
filePath = r'\\server01\d$\temp\Svc_Properties.txt'
doc = open(filePath, 'w')
header = "Server;Service;Status;MinIns;MaxIns;MaxIdle;MaxStart;MaxUsage;MaxWait;IsolationLevel;Cluster"
doc.write(header +"\n")

# ###########################################################################################################################
# End of Variables
# ###########################################################################################################################
# Print some info
print
print "This script is to collect all services' information."
serverPort = 6080
print

# Create a list to hold folder names
folderList = []

# Create a list to hold stopped/started services
targetList = []

# Get a token
token = getToken(username, password, serverName, serverPort,doc)
if token is None:
print "Could not generate a token with the username and password provided."
doc.write("\nCould not generate a token with the username and password provided." +"\n")
doc.close()
return

folderList = getCatalog(token,serverName,serverPort,doc)

for folder in folderList:
count = targetList.__len__()
i =0
folderName = folder
print "Processing folder : " + folderName
#doc.write("\n"+ "Processing folder : " + folderName + "\n")
while(i < count):
targetList.remove(targetList.__getitem__(0))
i = i +1

# Construct URL to read folder
if folder == "ROOT":
folder = ""
else:
folder += "/"

folderURL = "/arcgis/admin/services/" + folder

# This request only needs the token and the response formatting parameter
params = urllib.urlencode({'token': token, 'f': 'json'})

headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}

# Connect to URL and post parameters
httpConn = httplib.HTTPConnection(serverName, serverPort)
httpConn.request("POST", folderURL, params, headers)

# Read response
response = httpConn.getresponse()
if (response.status != 200):
httpConn.close()
print "Could not read folder information."
doc.write("\n" + "Could not read folder information." +"\n")
return
else:
data = response.read()

# Check that data returned is not an error object
if not assertJsonSuccess(data,doc):
print "Error when reading folder information. " + str(data)
#doc.write("\n" + "Error when reading folder information. " + str(data)+"\n")
else:
print "Processed folder information successfully. Now processing services..."
#doc.write("Processed folder information successfully. Now processing services..." +"\n")
print

# Deserialize response into Python object
dataObj = json.loads(data)
httpConn.close()

counter = 0

# Loop through each service in the folder and stop or start it
for item in dataObj['services']:

fullSvcName = item['serviceName'] + "." + item['type']

# Construct URL to stop or start service, then make the request
statusURL = "/arcgis/admin/services/" + folder + fullSvcName + "/status"
httpConn.request("POST", statusURL, params, headers)
# Read status response
statusResponse = httpConn.getresponse()
if (statusResponse.status != 200):
httpConn.close()
print "Error while checking status for " + fullSvcName
doc.write("\n" + "Error while checking status for " + fullSvcName +"\n")
return
else:
statusData = statusResponse.read()

# Check that data returned is not an error object
if not assertJsonSuccess(statusData,doc):
print "Error returned when retrieving status information for " + fullSvcName + "."
doc.write("\n" + "Error returned when retrieving status information for " + fullSvcName + "." +"\n")
print str(statusData)
doc.write("\n" + str(statusData) +"\n")

else:

statusDataObj = json.loads(statusData)
status = statusDataObj['realTimeState']

minInstURL = "/arcgis/admin/services/" + folder + fullSvcName
httpConn.request("POST", minInstURL, params, headers)

# Collect all services information, you can also query any information available in Service properties
# Read status response
minInstResponse = httpConn.getresponse()
minInstData = minInstResponse.read()
minInstDataObj = json.loads(minInstData)
minIns = minInstDataObj['minInstancesPerNode']
maxIns = minInstDataObj['maxInstancesPerNode']
maxIdle = minInstDataObj['maxIdleTime']
maxStr = minInstDataObj['maxStartupTime']
maxUsg = minInstDataObj['maxUsageTime']
maxWait = minInstDataObj['maxWaitTime']
isoLvl = minInstDataObj['isolationLevel']
cluster = minInstDataObj['clusterName']
print (serverName + ';'+ str( fullSvcName) + ';'+ str(status) + ';'+ str(minIns) +';' + str(maxIns) +';' +str(maxIdle) +';' +str(maxStr) +';' +str(maxUsg) +';' +str(maxWait) +';' +str(isoLvl) +';' +str(cluster) )
doc.write(serverName + ';'+ str( fullSvcName) + ';'+ str(status) +';'+ str(minIns) +';' + str(maxIns) +';' +str(maxIdle) +';' +str(maxStr) +';' +str(maxUsg) +';' +str(maxWait) +';' +str(isoLvl) +';' +str(cluster) +'\n' )
httpConn.close()

print
doc.close()
return

def getCatalog(token,server,port,doc):
baseUrl = "http://{}:{}/arcgis/admin/services".format(server, port)
folderNameList = []
catalog = json.load(urllib2.urlopen(baseUrl + "/" + "?f=json&token=" + token))
print ' List of Folders :'
#doc.write(" List of Folders :" +"\n")
print '\tROOT'
#doc.write("\tROOT" +"\n")
folderNameList.append("ROOT")
if "error" in catalog: return
folders = catalog['folders']
for folderName in folders:
folderNameList.append(folderName)
print '\t'+folderName
#doc.write("\t"+folderName +"\n")
if "error" in catalog: return
else: return folderNameList
# A function to generate a token given username, password and the adminURL.
def getToken(username, password, serverName, serverPort,doc):
# Token URL is typically http://server[:port]/arcgis/admin/generateToken
tokenURL = "/arcgis/admin/generateToken"

params = urllib.urlencode({'username': username, 'password': password, 'client': 'requestip', 'f': 'json'})

headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}

# Connect to URL and post parameters
httpConn = httplib.HTTPConnection(serverName, serverPort)
httpConn.request("POST", tokenURL, params, headers)

# Read response
response = httpConn.getresponse()
if (response.status != 200):
httpConn.close()
print "Error while fetching tokens from admin URL. Please check the URL and try again."
doc.write("\n" + "Error while fetching tokens from admin URL. Please check the URL and try again." +"\n")
return
else:
data = response.read()
httpConn.close()

# Check that data returned is not an error object
if not assertJsonSuccess(data,doc):
return

# Extract the token from it
token = json.loads(data)
return token['token']
# A function that checks that the input JSON object
# is not an error object.
def assertJsonSuccess(data,doc):
obj = json.loads(data)
if 'status' in obj and obj['status'] == "error":
print "Error: JSON object returns an error. " + str(obj)
#doc.write("\n" + "Error: JSON object returns an error. " + str(obj) +"\n")
return False
else:
return True
# Script start
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))

FINDING GHOST ARCSOC


Many times in many occasions, I came across with “Ghost ArcSOC”. What is Ghost ArcSOC? This is my term, it is basically ArcSOC process which you can see from Windows Task Manager, that running, consuming your server resources BUT the server is not part of any cluster. How come it happens? for my case, I have multiple servers in cluster, sometimes when I removed a server from a cluster, the ArcSOC process would disappear eventually but for some reasons, it wouldn’t. OR in another case for Ghost ArcSOC, one of your servers in a cluster didn’t crank up a ArcSOC process and caused your system degrades.

I created below Python script to make me easier and faster to identify this kind of issue.

</pre>

#-------------------------------------------------------------------------------
# Name: Finding_GHOST.py
# Purpose: List All Started Services, minimum instances and compare them against
# ArcSOC running on all servers
# This script could be used to find GHOST ArcSOC.GHOST ArcSOC is ArcSOC process that running on server
# but the service status is not running on ArcGIS Manager/Admin.
# If column FOUND = NO means ArcSOC process is running BUT Service status is STOPPED = GHOST is found
# Author: Oey
#
# Created: 09/15/2015
# Copyright: (c) Oey 2015
# Licence: <your licence>
#-------------------------------------------------------------------------------

# For Http calls
import httplib, urllib,json,urllib2

# For system tools
import sys, datetime

# For reading passwords without echoing
import getpass
import wmi

#Get the current date
now = datetime.datetime.now()

# Defines the entry point into the script
def main(argv=None):

# ##########################################################################################################################
# Start of Variables
# ##########################################################################################################################

# Server name - Pick a server name that the script will use to login on Admin console
serverName = 'ArcGIS1'

# list of Servers in environment
servers = ["ArcGIS1", "ArcGIS2","ArcGIS3"]

# Status of the Service
startStop = 'STARTED'
##startStop = 'STOPPED'

# Admin/publisher user name and password
username = 'admin'
password = 'admin_123'

# Log file location
filePath = r'\\ArcGIS1\d$\temp\Services_' + startStop + '_Services_' + str(now.year) + str(now.month) + str(now.day) + '.txt'
# ###########################################################################################################################
# End of Variables
# ###########################################################################################################################
doc = open(filePath, "w")
header = "Server,Service,Number of SOCs,Found\n"
doc.write(header)

# Print some info
print
print "This script detects " + startStop + " services in a folder."
print

serverPort = 6080
print

# Create a list to hold folder names
folderList = []

# Create a list to hold stopped/started services
targetList = []

# Get a token
token = getToken(username, password, serverName, serverPort,doc)
if token is None:
print "Could not generate a token with the username and password provided."
doc.write("\nCould not generate a token with the username and password provided." +"\n")
doc.close()
return

folderList = getCatalog(token,serverName,serverPort,doc)

for folder in folderList:
folderName = folder
print
print "Processing folder : " + folderName

# Construct URL to read folder
if folder == "ROOT":
folder = ""
else:
folder += "/"

folderURL = "/arcgis/admin/services/" + folder

# This request only needs the token and the response formatting parameter
params = urllib.urlencode({'token': token, 'f': 'json'})
headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}

# Connect to URL and post parameters
httpConn = httplib.HTTPConnection(serverName, serverPort)
httpConn.request("POST", folderURL, params, headers)

# Read response
response = httpConn.getresponse()
if (response.status != 200):
httpConn.close()
print "Could not read folder information."
doc.write("\n" + "Could not read folder information." +"\n")
return
else:
data = response.read()

# Check that data returned is not an error object
if not assertJsonSuccess(data,doc):
print "Error when reading folder information. " + str(data)
doc.write("\n" + "Error when reading folder information. " + str(data)+"\n")
else:
print "Processed folder information successfully. Now processing services..."
# Deserialize response into Python object
dataObj = json.loads(data)
httpConn.close()
counter = 0

# Loop through each service in the folder and stop or start it
for item in dataObj['services']:

fullSvcName = item['serviceName'] + "." + item['type']

# Construct URL to stop or start service, then make the request
statusURL = "/arcgis/admin/services/" + folder + fullSvcName + "/status"
httpConn.request("POST", statusURL, params, headers)

# Read status response
statusResponse = httpConn.getresponse()
if (statusResponse.status != 200):
httpConn.close()
print "Error while checking status for " + fullSvcName
doc.write("\n" + "Error while checking status for " + fullSvcName +"\n")
return
else:
statusData = statusResponse.read()

# Check that data returned is not an error object
if not assertJsonSuccess(statusData,doc):
print "Error returned when retrieving status information for " + fullSvcName + "."
doc.write("\n" + "Error returned when retrieving status information for " + fullSvcName + "." +"\n")
print str(statusData)
doc.write("\n" + str(statusData) +"\n")

else:
# Add the stopped/started service and the current time to a list
statusDataObj = json.loads(statusData)

if statusDataObj['realTimeState'] == startStop:

# Find MINIMUM INSTANCES PER NODE, IF 1 or more then count into it
minInstURL = "/arcgis/admin/services/" + folder + fullSvcName
httpConn.request("POST", minInstURL, params, headers)

# Read status response
minInstResponse = httpConn.getresponse()
minInstData = minInstResponse.read()
minInstDataObj = json.loads(minInstData)
if minInstDataObj['minInstancesPerNode'] >= 1:
fullSvcNames = folderName + "." + fullSvcName

# Append it into list
counter = counter + 1
targetList.append(fullSvcNames)
#targetList.append([fullSvcName + "(" + str(minInstDataObj['minInstancesPerNode']) + ")" ])

httpConn.close()

# Check number of stopped/started services found
if len(targetList) == 0:
print "No " + startStop + " services detected in folder " + str(folderName)
print
##doc.write("\n\t" + "No " + startStop + " services detected in folder " + str(folderName) +"\n")
##else:
# Write out all the stopped/started services found
# This could alternatively be written to an e-mail or a log file

# ##########################################################################################
# START OF - This portion is to extract ArcSOC process on each servers
# ##########################################################################################

for server in servers:

#Flush dictionary
hitDict = {}
c= wmi.WMI(server)
ArcSOC = 0
for process in c.Win32_Process ():
if process.Name == "ArcSOC.exe":
ArcSOC = ArcSOC + 1
strCmdLine = process.CommandLine

# The data is: "D:\Program Files\ArcGIS\Server\bin\ArcSOC.exe" -XX:-CreateMinidumpOnCrash -Xmx64M -Dservice=maps.AZR_TST.MapServer "-Djava.class.path=D:\Program Files\ArcGIS\
astrVerify = "-Dservice="
ostrVerify = "-Djava.class"
a = strCmdLine.find(astrVerify)
o = strCmdLine.find(ostrVerify)
keyCheck = strCmdLine[a+len(astrVerify):o]
if keyCheck in hitDict:
stats = hitDict[keyCheck]

# Add 1 to tally of hits
stats[0] += 1

else:
# Add key with one hit
hitDict[keyCheck] = [1]

# Read through dictionary
for key in sorted(hitDict):

# Compare each ArcSOC with List of STATUS that collected from Server Admin
if key in targetList:

# Calculate total hit
totalDraws = hitDict[key][0]

# Construct and write the comma-separated line
line = server +"," + key + "," + str(totalDraws) + ",YES" +"\n"
line2 = server +"," + key + "," + str(totalDraws) + ",YES"
print line2
doc.write(line)
else:

# Calculate total hit
totalDraws = hitDict[key][0]

# Construct and write the comma-separated line
line = server +"," + key + "," + str(totalDraws) +",NO" +"\n"
line2 = server +"," + key + "," + str(totalDraws) + ",NO"
print line2

doc.write(line)
# ##########################################################################################
# END OF - This portion is to extract ArcSOC process on each servers
# ##########################################################################################

doc.close()
return

def getCatalog(token,server,port,doc):
baseUrl = "http://{}:{}/arcgis/admin/services".format(server, port)
folderNameList = []
catalog = json.load(urllib2.urlopen(baseUrl + "/" + "?f=json&token=" + token))
print ' List of Folders :'
##doc.write(" List of Folders :" +"\n")
print '\tROOT'
##doc.write("\tROOT" +"\n")
folderNameList.append("ROOT")
if "error" in catalog: return
folders = catalog['folders']
for folderName in folders:
folderNameList.append(folderName)
print '\t'+folderName
##doc.write("\t"+folderName +"\n")
if "error" in catalog: return
else: return folderNameList
# A function to generate a token given username, password and the adminURL.
def getToken(username, password, serverName, serverPort,doc):
# Token URL is typically http://server[:port]/arcgis/admin/generateToken
tokenURL = "/arcgis/admin/generateToken"

params = urllib.urlencode({'username': username, 'password': password, 'client': 'requestip', 'f': 'json'})

headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}

# Connect to URL and post parameters
httpConn = httplib.HTTPConnection(serverName, serverPort)
httpConn.request("POST", tokenURL, params, headers)

# Read response
response = httpConn.getresponse()
if (response.status != 200):
httpConn.close()
print "Error while fetching tokens from admin URL. Please check the URL and try again."
doc.write("\n" + "Error while fetching tokens from admin URL. Please check the URL and try again." +"\n")
return
else:
data = response.read()
httpConn.close()

# Check that data returned is not an error object
if not assertJsonSuccess(data,doc):
return

# Extract the token from it
token = json.loads(data)
return token['token']
# A function that checks that the input JSON object
# is not an error object.
def assertJsonSuccess(data,doc):
obj = json.loads(data)
if 'status' in obj and obj['status'] == "error":
print "Error: JSON object returns an error. " + str(obj)
#doc.write("\n" + "Error: JSON object returns an error. " + str(obj) +"\n")
return False
else:
return True
# Script start
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
<pre>