Many times in many occasions, I came across with “Ghost ArcSOC”. What is Ghost ArcSOC? This is my term, it is basically ArcSOC process which you can see from Windows Task Manager, that running, consuming your server resources BUT the server is not part of any cluster. How come it happens? for my case, I have multiple servers in cluster, sometimes when I removed a server from a cluster, the ArcSOC process would disappear eventually but for some reasons, it wouldn’t. OR in another case for Ghost ArcSOC, one of your servers in a cluster didn’t crank up a ArcSOC process and caused your system degrades.
I created below Python script to make me easier and faster to identify this kind of issue.
</pre>
#-------------------------------------------------------------------------------
# Name: Finding_GHOST.py
# Purpose: List All Started Services, minimum instances and compare them against
# ArcSOC running on all servers
# This script could be used to find GHOST ArcSOC.GHOST ArcSOC is ArcSOC process that running on server
# but the service status is not running on ArcGIS Manager/Admin.
# If column FOUND = NO means ArcSOC process is running BUT Service status is STOPPED = GHOST is found
# Author: Oey
#
# Created: 09/15/2015
# Copyright: (c) Oey 2015
# Licence: <your licence>
#-------------------------------------------------------------------------------
# For Http calls
import httplib, urllib,json,urllib2
# For system tools
import sys, datetime
# For reading passwords without echoing
import getpass
import wmi
#Get the current date
now = datetime.datetime.now()
# Defines the entry point into the script
def main(argv=None):
# ##########################################################################################################################
# Start of Variables
# ##########################################################################################################################
# Server name - Pick a server name that the script will use to login on Admin console
serverName = 'ArcGIS1'
# list of Servers in environment
servers = ["ArcGIS1", "ArcGIS2","ArcGIS3"]
# Status of the Service
startStop = 'STARTED'
##startStop = 'STOPPED'
# Admin/publisher user name and password
username = 'admin'
password = 'admin_123'
# Log file location
filePath = r'\\ArcGIS1\d$\temp\Services_' + startStop + '_Services_' + str(now.year) + str(now.month) + str(now.day) + '.txt'
# ###########################################################################################################################
# End of Variables
# ###########################################################################################################################
doc = open(filePath, "w")
header = "Server,Service,Number of SOCs,Found\n"
doc.write(header)
# Print some info
print
print "This script detects " + startStop + " services in a folder."
print
serverPort = 6080
print
# Create a list to hold folder names
folderList = []
# Create a list to hold stopped/started services
targetList = []
# Get a token
token = getToken(username, password, serverName, serverPort,doc)
if token is None:
print "Could not generate a token with the username and password provided."
doc.write("\nCould not generate a token with the username and password provided." +"\n")
doc.close()
return
folderList = getCatalog(token,serverName,serverPort,doc)
for folder in folderList:
folderName = folder
print
print "Processing folder : " + folderName
# Construct URL to read folder
if folder == "ROOT":
folder = ""
else:
folder += "/"
folderURL = "/arcgis/admin/services/" + folder
# This request only needs the token and the response formatting parameter
params = urllib.urlencode({'token': token, 'f': 'json'})
headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
# Connect to URL and post parameters
httpConn = httplib.HTTPConnection(serverName, serverPort)
httpConn.request("POST", folderURL, params, headers)
# Read response
response = httpConn.getresponse()
if (response.status != 200):
httpConn.close()
print "Could not read folder information."
doc.write("\n" + "Could not read folder information." +"\n")
return
else:
data = response.read()
# Check that data returned is not an error object
if not assertJsonSuccess(data,doc):
print "Error when reading folder information. " + str(data)
doc.write("\n" + "Error when reading folder information. " + str(data)+"\n")
else:
print "Processed folder information successfully. Now processing services..."
# Deserialize response into Python object
dataObj = json.loads(data)
httpConn.close()
counter = 0
# Loop through each service in the folder and stop or start it
for item in dataObj['services']:
fullSvcName = item['serviceName'] + "." + item['type']
# Construct URL to stop or start service, then make the request
statusURL = "/arcgis/admin/services/" + folder + fullSvcName + "/status"
httpConn.request("POST", statusURL, params, headers)
# Read status response
statusResponse = httpConn.getresponse()
if (statusResponse.status != 200):
httpConn.close()
print "Error while checking status for " + fullSvcName
doc.write("\n" + "Error while checking status for " + fullSvcName +"\n")
return
else:
statusData = statusResponse.read()
# Check that data returned is not an error object
if not assertJsonSuccess(statusData,doc):
print "Error returned when retrieving status information for " + fullSvcName + "."
doc.write("\n" + "Error returned when retrieving status information for " + fullSvcName + "." +"\n")
print str(statusData)
doc.write("\n" + str(statusData) +"\n")
else:
# Add the stopped/started service and the current time to a list
statusDataObj = json.loads(statusData)
if statusDataObj['realTimeState'] == startStop:
# Find MINIMUM INSTANCES PER NODE, IF 1 or more then count into it
minInstURL = "/arcgis/admin/services/" + folder + fullSvcName
httpConn.request("POST", minInstURL, params, headers)
# Read status response
minInstResponse = httpConn.getresponse()
minInstData = minInstResponse.read()
minInstDataObj = json.loads(minInstData)
if minInstDataObj['minInstancesPerNode'] >= 1:
fullSvcNames = folderName + "." + fullSvcName
# Append it into list
counter = counter + 1
targetList.append(fullSvcNames)
#targetList.append([fullSvcName + "(" + str(minInstDataObj['minInstancesPerNode']) + ")" ])
httpConn.close()
# Check number of stopped/started services found
if len(targetList) == 0:
print "No " + startStop + " services detected in folder " + str(folderName)
print
##doc.write("\n\t" + "No " + startStop + " services detected in folder " + str(folderName) +"\n")
##else:
# Write out all the stopped/started services found
# This could alternatively be written to an e-mail or a log file
# ##########################################################################################
# START OF - This portion is to extract ArcSOC process on each servers
# ##########################################################################################
for server in servers:
#Flush dictionary
hitDict = {}
c= wmi.WMI(server)
ArcSOC = 0
for process in c.Win32_Process ():
if process.Name == "ArcSOC.exe":
ArcSOC = ArcSOC + 1
strCmdLine = process.CommandLine
# The data is: "D:\Program Files\ArcGIS\Server\bin\ArcSOC.exe" -XX:-CreateMinidumpOnCrash -Xmx64M -Dservice=maps.AZR_TST.MapServer "-Djava.class.path=D:\Program Files\ArcGIS\
astrVerify = "-Dservice="
ostrVerify = "-Djava.class"
a = strCmdLine.find(astrVerify)
o = strCmdLine.find(ostrVerify)
keyCheck = strCmdLine[a+len(astrVerify):o]
if keyCheck in hitDict:
stats = hitDict[keyCheck]
# Add 1 to tally of hits
stats[0] += 1
else:
# Add key with one hit
hitDict[keyCheck] = [1]
# Read through dictionary
for key in sorted(hitDict):
# Compare each ArcSOC with List of STATUS that collected from Server Admin
if key in targetList:
# Calculate total hit
totalDraws = hitDict[key][0]
# Construct and write the comma-separated line
line = server +"," + key + "," + str(totalDraws) + ",YES" +"\n"
line2 = server +"," + key + "," + str(totalDraws) + ",YES"
print line2
doc.write(line)
else:
# Calculate total hit
totalDraws = hitDict[key][0]
# Construct and write the comma-separated line
line = server +"," + key + "," + str(totalDraws) +",NO" +"\n"
line2 = server +"," + key + "," + str(totalDraws) + ",NO"
print line2
doc.write(line)
# ##########################################################################################
# END OF - This portion is to extract ArcSOC process on each servers
# ##########################################################################################
doc.close()
return
def getCatalog(token,server,port,doc):
baseUrl = "http://{}:{}/arcgis/admin/services".format(server, port)
folderNameList = []
catalog = json.load(urllib2.urlopen(baseUrl + "/" + "?f=json&token=" + token))
print ' List of Folders :'
##doc.write(" List of Folders :" +"\n")
print '\tROOT'
##doc.write("\tROOT" +"\n")
folderNameList.append("ROOT")
if "error" in catalog: return
folders = catalog['folders']
for folderName in folders:
folderNameList.append(folderName)
print '\t'+folderName
##doc.write("\t"+folderName +"\n")
if "error" in catalog: return
else: return folderNameList
# A function to generate a token given username, password and the adminURL.
def getToken(username, password, serverName, serverPort,doc):
# Token URL is typically http://server[:port]/arcgis/admin/generateToken
tokenURL = "/arcgis/admin/generateToken"
params = urllib.urlencode({'username': username, 'password': password, 'client': 'requestip', 'f': 'json'})
headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
# Connect to URL and post parameters
httpConn = httplib.HTTPConnection(serverName, serverPort)
httpConn.request("POST", tokenURL, params, headers)
# Read response
response = httpConn.getresponse()
if (response.status != 200):
httpConn.close()
print "Error while fetching tokens from admin URL. Please check the URL and try again."
doc.write("\n" + "Error while fetching tokens from admin URL. Please check the URL and try again." +"\n")
return
else:
data = response.read()
httpConn.close()
# Check that data returned is not an error object
if not assertJsonSuccess(data,doc):
return
# Extract the token from it
token = json.loads(data)
return token['token']
# A function that checks that the input JSON object
# is not an error object.
def assertJsonSuccess(data,doc):
obj = json.loads(data)
if 'status' in obj and obj['status'] == "error":
print "Error: JSON object returns an error. " + str(obj)
#doc.write("\n" + "Error: JSON object returns an error. " + str(obj) +"\n")
return False
else:
return True
# Script start
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
<pre>
MyArcGISserver 