Skip to content
GitLab
About GitLab
GitLab: the DevOps platform
Explore GitLab
Install GitLab
How GitLab compares
Get started
GitLab docs
GitLab Learn
Pricing
Talk to an expert
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Projects
Groups
Snippets
Sign up now
Login
Sign in
Toggle navigation
Menu
Open sidebar
Video AG Infrastruktur
jobmanager
Commits
d1bc9ae7
Commit
d1bc9ae7
authored
Dec 29, 2017
by
Julian Rother
Browse files
Extended jobmanager to support default and background queue
parent
02bd0d08
Changes
1
Hide whitespace changes
Inline
Side-by-side
manager.py
View file @
d1bc9ae7
...
...
@@ -22,8 +22,8 @@ class WorkerApi(object):
r
=
requests
.
post
(
self
.
baseurl
+
'/internal/jobs/api/worker/'
+
gethostname
()
+
'/ping'
,
params
=
{
'apikey'
:
self
.
apikey
})
return
r
.
status_code
==
200
def
worker_schedule
(
self
,
jobtypes
):
r
=
requests
.
post
(
self
.
baseurl
+
'/internal/jobs/api/worker/'
+
gethostname
()
+
'/schedule'
,
json
=
{
'apikey'
:
self
.
apikey
,
'jobtypes'
:
","
.
join
(
jobtypes
)
})
def
worker_schedule
(
self
,
jobtypes
,
queues
):
r
=
requests
.
post
(
self
.
baseurl
+
'/internal/jobs/api/worker/'
+
gethostname
()
+
'/schedule'
,
json
=
{
'apikey'
:
self
.
apikey
,
'jobtypes'
:
jobtypes
,
'queues'
:
queues
})
if
r
.
status_code
==
200
:
return
r
.
json
()
else
:
...
...
@@ -82,23 +82,44 @@ def get_jobtypes():
res
.
append
(
name
)
return
res
procs
=
[]
proc_queues
=
{
'default'
:
{},
'background'
:
{}}
queue_sizes
=
{
'default'
:
int
(
os
.
environ
.
get
(
"WORKER_DEFAULT_QUEUE"
,
"2"
)),
'background'
:
int
(
os
.
environ
.
get
(
"WORKER_BACKGROUND_QUEUE"
,
"2"
))}
def
pop_pidinfo
(
pid
):
for
name
,
queue
in
proc_queues
.
items
():
if
pid
in
queue
:
return
queue
.
pop
(
pid
)
return
None
while
True
:
for
p
in
procs
:
p
.
poll
()
if
p
.
returncode
==
None
:
continue
p
.
wait
()
procs
.
remove
(
p
)
if
psutil
.
cpu_percent
()
>
85
:
time
.
sleep
(
0.1
)
while
max
(
map
(
len
,
proc_queues
.
values
())):
pid
,
status
=
os
.
waitpid
(
0
,
os
.
WNOHANG
)
if
not
pid
:
break
info
=
pop_pidinfo
(
pid
)
if
status
!=
0
:
print
(
'Job %i (pid %i) terminated with non-zero exit code %i'
%
(
info
,
pid
,
status
))
queues
=
[]
for
name
,
queue
in
proc_queues
.
items
():
if
len
(
queue
)
<
queue_sizes
[
name
]:
queues
.
append
(
name
)
if
not
queues
:
time
.
sleep
(
30
)
continue
j
=
api
.
worker_schedule
(
get_jobtypes
())
j
=
api
.
worker_schedule
(
get_jobtypes
(),
queues
)
if
not
j
:
time
.
sleep
(
30
)
continue
print
(
"started jobid %i"
%
j
[
'id'
])
if
str
(
j
[
'type'
])
in
get_jobtypes
():
procs
.
append
(
subprocess
.
Popen
([
workerdir
+
'/'
+
str
(
j
[
'type'
]),
str
(
j
[
'id'
]),
str
(
j
[
'type'
]),
str
(
j
[
'priority'
])
,
str
(
j
[
'data'
])
]
))
if
str
(
j
[
'type'
])
in
get_jobtypes
()
and
str
(
j
[
'queue'
])
in
queues
:
path
=
workerdir
+
'/'
+
str
(
j
[
'type'
])
print
(
"Started Job %i"
%
j
[
'id'
])
pid
=
os
.
spawnv
(
os
.
P_NOWAIT
,
path
,
[
path
,
str
(
j
[
'id'
]),
str
(
j
[
'type'
]),
str
(
j
[
'priority'
]),
str
(
j
[
'data'
])])
proc_queues
[
str
(
j
[
'queue'
])][
pid
]
=
j
[
'id'
]
if
str
(
j
[
'queue'
])
==
'background'
:
os
.
sched_setscheduler
(
pid
,
os
.
SCHED_IDLE
,
os
.
sched_param
(
0
))
time
.
sleep
(
1
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment