Skip to content

Commit

Permalink
Adds stickiness to browsing deep links within a site
Browse files Browse the repository at this point in the history
  • Loading branch information
@sei-dupdyke
sei-dupdyke committed Mar 18, 2020
1 parent c244cf1 commit aa0d10f
Show file tree
Hide file tree
Showing 8 changed files with 289 additions and 59 deletions.
67 changes: 39 additions & 28 deletions Ghosts.Client/Handlers/BaseBrowserHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ public abstract class BaseBrowserHandler : BaseHandler
public static readonly Logger _log = LogManager.GetCurrentClassLogger();
public IWebDriver Driver { get; set; }
public HandlerType BrowserType { get; set; }
private int _stickiness = 0;
private int _depthMin = 1;
private int _depthMax = 10;

public void ExecuteEvents(TimelineHandler handler)
{
Expand All @@ -39,20 +42,18 @@ public void ExecuteEvents(TimelineHandler handler)
{
case "random":

var stickiness = 0;
var depthMin = 1;
var depthMax = 10;
// setup
if (handler.HandlerArgs.ContainsKey("stickiness"))
{
int.TryParse(handler.HandlerArgs["stickiness"], out stickiness);
int.TryParse(handler.HandlerArgs["stickiness"], out _stickiness);
}
if (handler.HandlerArgs.ContainsKey("stickiness-depth-min"))
{
int.TryParse(handler.HandlerArgs["stickiness-depth-min"], out depthMin);
int.TryParse(handler.HandlerArgs["stickiness-depth-min"], out _depthMin);
}
if (handler.HandlerArgs.ContainsKey("stickiness-depth-max"))
{
int.TryParse(handler.HandlerArgs["stickiness-depth-max"], out depthMax);
int.TryParse(handler.HandlerArgs["stickiness-depth-max"], out _depthMax);
}

while (true)
Expand All @@ -68,49 +69,59 @@ public void ExecuteEvents(TimelineHandler handler)
MakeRequest(config);
Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), timelineEvent.TrackableId);

if (stickiness > 0)
if (this._stickiness > 0)
{
var random = new Random();
//now some percentage of the time should stay on this site
if (random.Next(100) < stickiness)
if (random.Next(100) < this._stickiness)
{
var loops = random.Next(depthMin, depthMax);
var loops = random.Next(this._depthMin, this._depthMax);
for (var loopNumber = 0; loopNumber < loops; loopNumber++)
{
try
{
var linkManager = new LinkManager(config.GetHost());


//get all links
var links = Driver.FindElements(By.TagName("a"));
if (links.Count > 0)
foreach (var l in links)
{
var linkSelected = random.Next(links.Count);
var href = links[linkSelected].GetAttribute("href");
while (!href.StartsWith("http"))
var node = l.GetAttribute("href");
if (string.IsNullOrEmpty(node) ||
node.ToLower().StartsWith("//"))
{
foreach (var l in links)
{
href = l.GetAttribute("href");
}
//skip, these seem ugly
}

if (!string.IsNullOrEmpty(href))
// http|s links
else if (node.ToLower().StartsWith("http"))
{
if (!href.StartsWith("http"))
{
href = $"http://{href}";
}
config.Method = "GET";
config.Uri = new Uri(href);

MakeRequest(config);
Report(handler.HandlerType.ToString(), timelineEvent.Command,config.ToString(), timelineEvent.TrackableId);
linkManager.AddLink(node.ToLower());
}
// relative links - prefix the scheme and host
else
{
linkManager.AddLink($"{config.GetHost()}{node.ToLower()}");
}
}

var link = linkManager.Choose();
if (link == null)
{
return;
}

config.Method = "GET";
config.Uri = link.Url;

MakeRequest(config);
Report(handler.HandlerType.ToString(), timelineEvent.Command, config.ToString(), timelineEvent.TrackableId);
}
catch (Exception e)
{
_log.Error(e);
}

Thread.Sleep(timelineEvent.DelayAfter);
}
}
Expand Down
5 changes: 5 additions & 0 deletions Ghosts.Client/Infrastructure/Browser/RequestConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ namespace Ghosts.Client.Infrastructure.Browser
{
public class RequestConfiguration
{
public string GetHost()
{
return $"{this.Uri.Scheme}://{this.Uri.Host}";
}

/// <summary>
/// URL to browse
/// </summary>
Expand Down
6 changes: 3 additions & 3 deletions Ghosts.Client/config/timeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,9 +140,9 @@
"blockstyles": "true",
"blockflash": "true",
"blockscripts": "true",
"stickiness": 80,
"stickiness-depth-min": 1,
"stickiness-depth-max": 10
"stickiness": 75,
"stickiness-depth-min": 5,
"stickiness-depth-max": 10000
},
"Initial": "about:blank",
"UtcTimeOn": "00:00:00",
Expand Down
84 changes: 84 additions & 0 deletions Ghosts.Domain/Code/LinkManager.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Copyright 2017 Carnegie Mellon University. All Rights Reserved. See LICENSE.md file for terms.

using System;
using System.Collections.Generic;
using System.Linq;

namespace Ghosts.Domain.Code
{
public class Link
{
public Uri Url { get; set; }
public int Priority { get; set; }

public Link()
{
Priority = 0;
}
}

public class LinkManager
{
public List<Link> Links { private set; get; }
private readonly Random _random = new Random();
private readonly string _baseUrl;

public LinkManager(string baseUrl)
{
Links = new List<Link>();
this._baseUrl = baseUrl;
}

/// <summary>
/// Adds proper links — invalid links get quickly discarded
/// </summary>
/// <param name="url">http|s://some.link/path/etc</param>
public void AddLink(string url)
{
try
{
this.Links.Add(new Link {Url = new Uri(url)});
}
catch { }
}

public Link Choose()
{
var baseUri = new Uri(this._baseUrl);
foreach(var link in this.Links)
{
try
{
if (!link.Url.Host.Replace("www.","").Contains(baseUri.Host.Replace("www.","")))
link.Priority += 10;
}
catch (Exception e)
{
Console.WriteLine($"{link.Url} : {e}");
}
}

this.Links = this.Links.OrderByDescending(o=>o.Priority).ToList();

if (this.Links.Count < 1)
return null;

var totalWeight = Convert.ToInt32(this.Links.Sum(o => o.Priority));

// totalWeight is the sum of all weights
var r = _random.Next(0, totalWeight);

foreach (var link in this.Links)
{
if (r < link.Priority)
{
return link;
}

r -= link.Priority;
}

return Links.PickRandom();
}
}
}
2 changes: 2 additions & 0 deletions ghosts.client.linux/Handlers/BaseHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ public void Report(string handler, string command, string arg, string trackable)
result.CommandArg = arg;

if (!string.IsNullOrEmpty(trackable))
{
result.TrackableId = trackable;
}

var o = JsonConvert.SerializeObject(result,
Formatting.None,
Expand Down
Loading

0 comments on commit aa0d10f

Please sign in to comment.