c. level = (int) reader ["level"];

c. name = (string) reader ["name"];

c. url = (string) reader ["url"];

c. parentURL = (string) reader ["parentUrl"];

return c;

}

protected void GetCommandParameters (SqlParameterCollection parameters)

{

parameters. Clear ();

parameters. Add ("@id", SqlDbType. Int);

parameters. Add ("@level", SqlDbType. Int);

parameters. Add ("@name", SqlDbType. VarChar);

parameters. Add ("@url", SqlDbType. VarChar);

parameters. Add ("@parentUrl", SqlDbType. VarChar);

}

protected void FillCommandParameters (SqlParameterCollection parameters, Category c)

{

parameters ["@id"]. Value = c. id;

parameters ["@level"]. Value = c. level;

parameters ["@name"]. Value = c. name;

parameters ["@url"]. Value = c. url;

parameters ["@parentUrl"]. Value = c. parentURL;

}

#endregion

}

}

Модуль Grabber

using System;

using System. Collections. Generic;

using System. Linq;

using System. Text;

using System.net;

using System. IO;

using System. Windows. Forms;

using System. Text. RegularExpressions;

using System. Threading;

using System. Diagnostics;

using grabberMolotok. Threading;

namespace grabberMolotok

{

class Grabber

{

DBEngine. DBEngine dbEngine; // движок БД

List<Category> categories; // категории

int getItemCounter; // счетчик получения товаров

int getItemWorkAmount; // объём работы получения товаров

int getItemThreadCount; // количество потоков получения товаров

public int GetItemThreadCount

{

get { return getItemThreadCount; }

set { getItemThreadCount = value; }

}

int saveImageCounter; // счетчик сохранения изображений

int saveItemWorkAmount; // объём работы сохранения изображений

int saveItemThreadCount; // количество потоков сохранения изображений

public int SaveItemThreadCount

{

get { return saveItemThreadCount; }

set { saveItemThreadCount = value; }

}

#region События

public delegate void AllCategoriesParseCompletedEventHandler ();

public static event AllCategoriesParseCompletedEventHandler AllCategoriesParseCompletedEvent;

public delegate void AllCategoriesImageSaveCompletedEventHandler ();

public static event AllCategoriesImageSaveCompletedEventHandler AllCategoriesImageSaveCompletedEvent;

public delegate void CategoriesGotEventHandler ();

public static event CategoriesGotEventHandler CategoriesGotEvent;

void OnCategoryParseComplete ()

{

getItemCounter++;

if (getItemCounter == getItemWorkAmount)

{

AllCategoriesParseCompletedEvent ();

}

}

void OnCategoryImageSaveComplete ()

{

saveImageCounter++;

if (saveImageCounter == saveItemWorkAmount)

{

if (AllCategoriesImageSaveCompletedEvent! = null)

AllCategoriesImageSaveCompletedEvent ();

}

}

#endregion События

public Grabber ()

{

Category. CategoryParseCompletedEvent += new Category. CategoryParseCompletedEventHandler (this. OnCategoryParseComplete);

Category. CategoryImageSaveCompletedEvent += new Category. CategoryImageSaveCompletedEventHandler (this. OnCategoryImageSaveComplete);

dbEngine = new DBEngine. DBEngine ();

categories = new List<Category> ();

getItemCounter = 0;

getItemWorkAmount = - 1;

getItemThreadCount = 8;

saveImageCounter = 0;

saveItemWorkAmount = - 1;

saveItemThreadCount = 8;

}

// / <summary>

// / Получает список категорий с молотка

// / </summary>

public void GetCategories ()

{

Debug. WriteLine ("Получение категорий с сайта");

WebBrowser wB = new WebBrowser ();

wB. ScriptErrorsSuppressed = true;

string url = "http://molotok.ru/category_map. php";

wB. NavigateFromRequest (url);

List<HtmlElement> columnDivList = null;

List<HtmlElement> categoriesHtmlList = new List<HtmlElement> ();

columnDivList = wB. WaitElement ("pagecontent1", 10000)

. GetInnerElements ("table", 0)

. GetInnerElements ("tbody", 0)

. GetInnerElements ("tr",

3)

. GetInnerElements ("td", 1)

. GetInnerElements ("div");

foreach (var columnDiv in columnDivList)

{

categoriesHtmlList. AddRange (columnDiv. GetInnerElements ("div"));

}

Debug. WriteLine ("Категории получены с сайта");

ParseCategories (categoriesHtmlList);

if (CategoriesGotEvent! = null)

CategoriesGotEvent ();

}

// / <summary>

// / Парсит полученные с молотка категории

// / </summary>

// / <param name="categoriesHtmlList"></param>

void ParseCategories (List<HtmlElement> categoriesHtmlList)

{

Debug. WriteLine ("Начался парс категорий");

string [] LastLevelURL = new string [4];

string catName;

string catURL;

string catClass;

int catLevel;

int catSiteId;

Stopwatch wholeStopwatch = Stopwatch. StartNew ();

foreach (var categoryHtml in categoriesHtmlList)

{

// Stopwatch stopwatch = Stopwatch. StartNew ();

catName = GetName (categoryHtml);

catURL = GetURL (categoryHtml);

catClass = GetClass (categoryHtml);

catLevel = GetLevel (catClass);

catSiteId = GetSiteID (categoryHtml);

Category c = null;

switch (catLevel)

{

case 0:

{

c = new Category (catURL, "this is root", catName, 0, catLevel);

LastLevelURL [0] = catURL;

categories. Add (c);

break;

}

case 1:

{

c = new Category (catURL, LastLevelURL [0], catName, catSiteId, catLevel);

LastLevelURL [1] = catURL;

categories. Add (c);

break;

}

case 2:

{

c = new Category (catURL, LastLevelURL [1], catName, catSiteId, catLevel);

LastLevelURL [2] = catURL;

categories. Add (c);

break;

}

case 3:

{

c = new Category (catURL, LastLevelURL [2], catName, catSiteId, catLevel);

LastLevelURL [3] = catURL;

categories. Add (c);

break;

}

case 4:

{

c = new Category (catURL, LastLevelURL [3], catName, catSiteId, catLevel);

categories. Add (c);

break;

}

default: { break; }

}

// stopwatch. Stop ();

// Debug. WriteLine ( ( (double) stopwatch. ElapsedTicks / (double) Stopwatch. Frequency). ToString ());

}

wholeStopwatch. Stop ();

Debug. WriteLine ("Время парса всех категорий " + ( (double) wholeStopwatch. ElapsedTicks / (double) Stopwatch. Frequency). ToString () + "с");

Debug. WriteLine ("Парс категорий окончен");

}

#region Методы DBEngine

public void ClearTables ()

{

dbEngine. ClearTables ();

}

public void ClearCategories ()

{

dbEngine. categoryTable. Clear ();

}

public string ReadSetting (string settingName)

{

return dbEngine. settingsTable. ReadSetting (settingName);

}

// / <summary>

// / Сохраняет ссылки на изображения в БД

// / </summary>

// / <param name="c"></param>

public void SaveImages (Category c)

{

dbEngine. imageTable. Insert (c. categoryImages);

}

// / <summary>

// / Загружает ссылки на изображения из БД

// / </summary>

// / <param name="c"></param>

public void LoadImages (Category c)

{

c. categoryImages = dbEngine. imageTable. GetImagesInCategory (c);

}

// / <summary>

// / Сохраняет категории в БД

// / </summary>

public void SaveCategories ()

{

dbEngine. categoryTable. Insert (categories);

}

public Category GetCategoryById (int id)

{

return dbEngine. categoryTable. GetById (id);

}

// / <summary>

// / Загружает категории из БД

// / </summary>

public void LoadCategories ()

{

categories = dbEngine. categoryTable. GetAll ();

if (CategoriesGotEvent! = null)

CategoriesGotEvent ();

}

#endregion

#region Работа со списком категорий

// / <summary>

// / Возвращает список дочерних категорий (рекурсивная)

// / </summary>

// / <param name="root">Начальная категория</param>

// / <returns></returns>

public List<Category> GetAllChildCategories (Category root)

{

List<Category> childs = new List<Category> ();

childs. AddRange (

categories. FindAll (

delegate (Category category)

{

return category. parentURL == root. url;

}

)

);

int count = childs. Count;

if (count > 0 && root. level < 4)

{

for (int i = 0; i < count; i++)

{

childs. AddRange (GetAllChildCategories (childs [i]));

}

}

return childs;

}

// / <summary>

// / Возвращает список дочерних категорий

// / </summary>

// / <param name="root"></param>

// / <returns></returns>

public List<Category> GetChildCategories (Category root)

{

List<Category> childs = new List<Category> ();

childs. AddRange (

categories. FindAll (

delegate (Category category)

{

return category. parentURL == root. url;

}

)

);

return childs;

}

// / <summary>

// / Возвращает категории уровня 0

// / </summary>

// / <returns></returns>

public List<Category> GetRoots ()

{

List<Category> roots = new List<Category> ();

roots. AddRange (

categories. FindAll (

delegate (Category category)

{

return category. level == 0;

}

)

);

return roots;

}

#endregion

#region Вспомогательные функции

string GetName (HtmlElement e)

{

if (e. FirstChild! = null && e. FirstChild. FirstChild! = null) return e. FirstChild. FirstChild. InnerText;

return "";

}

string GetURL (HtmlElement e)

{

if (e. FirstChild! = null) return e. FirstChild. GetAttribute ("href"). Replace ("about:", "http://molotok.ru");

return "";

}

string GetClass (HtmlElement e)

{

Regex classExpression = new Regex (@"class=. {1} ([\w\d\s] *)", RegexOptions. IgnoreCase | RegexOptions.compiled);

Match match = classExpression. Match (e. OuterHtml);

if (match. Success)

{

return match. Groups [1]. Value;

}

return "";

}

int GetLevel (string classLevel)

{

Regex classExpression = new Regex (@" (\d{1})", RegexOptions. IgnoreCase | RegexOptions.compiled);

Match match = classExpression. Match (classLevel);

if (match. Success)

{

return Convert. ToInt32 (match. Groups [1]. Value);

}

return - 1;

}

int GetSiteID (HtmlElement e)

{

Regex siteIdExpression = new Regex (@" (\d{5,})", RegexOptions. IgnoreCase | RegexOptions.compiled);

Match match = siteIdExpression. Match (e. OuterHtml);

if (match. Success)

{

return Convert. ToInt32 (match. Groups [1]. Value);

}

return - 1;

}

#endregion

// / <summary>

// / Получает изображения в категории, в несколько потоков

// / </summary>

// / <param name="cats"></param>

public void ParallelGetItems (List<Category> cats)

{

Debug. WriteLine ("Запуск потоков парса ссылок на изображения");

this. getItemCounter = 0;

this. getItemWorkAmount = cats. Count;

var preparedList = ThreadHelper. SplitList (cats, getItemThreadCount);

foreach (var categories in preparedList)

{

Thread thread = new Thread (

delegate ()

{

foreach (var cat in categories)

{

cat. GetItems ();

Thread. Sleep (10);

}

}

);

thread. SetApartmentState (ApartmentState. STA);

thread. Start ();

thread. Join (10);

}

}

public void ParallelSaveImagesToDisk (List<Category> cats)

{

Debug. WriteLine ("Запуск потоков закачки изображений");

this. saveImageCounter = 0;

this. saveItemWorkAmount = cats. Count;

string initialDirectory = ReadSetting ("initialDirectory");

var preparedList = ThreadHelper. SplitList (cats, saveItemThreadCount);

foreach (var categories in preparedList)

{

Thread thread = new Thread (

delegate ()

{

foreach (var cat in categories)

{

cat. SaveImages (initialDirectory);

Thread. Sleep (10);

}

}

);

thread. SetApartmentState (ApartmentState. STA);

thread. Start ();

thread. Join (10);

}

}

}

}

using System;

using System. Collections. Generic;

using System. Linq;

using System. Text;

using System.net;

using System. IO;

using System. Text. RegularExpressions;

using System. Windows. Forms;

using System. Threading;

using System. Diagnostics;

namespace grabberMolotok

{

public class Category

{

public int id; // ИД категории в системе сайта

public int level; // Уровень вложения

public string name; // Название

public string url; // ссылка

public string parentURL; // ссылка на категорию-родитель

public List<Image> categoryImages; // изображения товаров

#region События

public delegate void CategoryParseCompletedEventHandler ();

public static event CategoryParseCompletedEventHandler CategoryParseCompletedEvent;

public delegate void CategoryImageSaveCompletedEventHandler ();

public static event CategoryImageSaveCompletedEventHandler CategoryImageSaveCompletedEvent;

#endregion

#region Конструкторы

public Category (string url, string parentURL, string name, int id, int level)

{

this. url = url;

this. parentURL = parentURL;

this. name = name;

this. id = id;

this. level = level;

categoryImages = new List<Image> ();

}

public Category (string url)

{

this. id = - 1;

this. level = - 1;

this. name = null;

this. url = url;

this. parentURL = null;

categoryImages = new List<Image> ();

}

public Category ()

{

this. id = - 1;

this. level = - 1;

this. name = null;

this. url = null;

this. parentURL = null;

categoryImages = new List<Image> ();

}

#endregion

// / <summary>

// / Получает список товаров в категории

// / </summary>

public void GetItems ()

{

WebBrowser wB = new WebBrowser ();

wB. ScriptErrorsSuppressed = true;

int pageCount = 0;

List<HtmlElement> htmlItems = new List<HtmlElement> ();

string url = this. url;

wB. NavigateFromRequest (url);

try

{

HtmlElement htmlPageCount = wB. WaitElement ("tabMainBox", 20000)

. GetInnerElements ("div", 0)

. GetInnerElements ("div", 0)

. GetInnerElements ("div", 0)

. GetInnerElements ("span", 1);

Regex pageCountRegex = new Regex (@"< [\D] span> (\d*) < [\D] span>", RegexOptions. IgnoreCase | RegexOptions.compiled);

Match match = pageCountRegex. Match (htmlPageCount. OuterHtml);

if (match. Success)

{

pageCount = Convert. ToInt32 (match. Groups [1]. Value);

}

}

catch (NullReferenceException)

{

pageCount = 1;

}

for (int i = 1; i <= pageCount; i++)

{

for (int tries = 0; tries < 5; tries++)

{

try

{

// Stopwatch watch = Stopwatch. StartNew ();

wB. NavigateFromRequest ("http://molotok.ru/listing. php/showcat? id=" + id. ToString () + "&p=" + i. ToString ());

htmlItems. AddRange (wB. WaitElement ("tabMainBox", 20000)

. GetInnerElements ("div", 1)

. GetInnerElements ("table", 0)

. GetInnerElements ("tbody", 0)

. GetInnerElements ("tr")

);

// watch. Stop ();

// Debug. WriteLine ( ( (double) watch. ElapsedTicks / (double) Stopwatch. Frequency). ToString ());

break;

}

catch (NullReferenceException)

{

Thread. Sleep (50);

}

}

}

ParseItems (htmlItems);

CategoryParseCompletedEvent ();

}

// / <summary>

// / Парсит товары в категории

// / </summary>

// / <param name="itemsHtmlList"></param>

// / <param name="cat"></param>

void ParseItems (List<HtmlElement> itemsHtmlList)

{

Debug. WriteLine ("Начался парс товаров");

Regex itemIdRegex = new Regex ("data-id=\" ([\\d] *)", RegexOptions. IgnoreCase | RegexOptions.compiled);

Regex smallPicturesRegex = new Regex (@" (http://[\w\d] *. allegroimg. pl/photos/64x48 [/\d] * [_\d] *)", RegexOptions. IgnoreCase | RegexOptions.compiled);

Regex bigPicturesRegex = new Regex (@" (http://[\w\d] *. allegroimg. pl/photos/400x300 [/\d] * [_\d] *)", RegexOptions. IgnoreCase | RegexOptions.compiled);

foreach (var item in itemsHtmlList)

{

// Stopwatch watch = Stopwatch. StartNew ();

/************************************/

Int64 itemId = - 1;

List<Image> imgList = new List<Image> ();;

Match matchId = itemIdRegex. Match (item. OuterHtml);

if (matchId. Success)

{

itemId = Convert. ToInt64 (matchId. Groups [1]. Value);

}

Match matchsmallPictures = smallPicturesRegex. Match (item. OuterHtml);

while (matchsmallPictures. Success)

{

this. AddImage (itemId, matchsmallPictures. Groups [1]. Value, false);

matchsmallPictures = matchsmallPictures. NextMatch ();

}

Match matchbigPictures = bigPicturesRegex. Match (item. OuterHtml);

while (matchbigPictures. Success)

{

this. AddImage (itemId, matchbigPictures. Groups [1]. Value, true);

matchbigPictures = matchbigPictures. NextMatch ();

}

/********************************************/

// watch. Stop ();

// Debug. WriteLine ( ( (double) watch. ElapsedTicks / (double) Stopwatch. Frequency). ToString ());

}

}

public void AddImage (Int64 itemId, string thumbnailURL, bool isBig)

{

Image img = new Image (itemId, id, thumbnailURL, isBig);

categoryImages. Add (img);

}

public void SaveImages (string initialDirectory)

{

foreach (var image in categoryImages)

{

Stopwatch watch = Stopwatch. StartNew ();

image. Save (initialDirectory);

watch. Stop ();

Debug. WriteLine ( ( (double) watch. ElapsedTicks / (double) Stopwatch. Frequency). ToString ());

}

CategoryImageSaveCompletedEvent ();

}

}

}

Модуль NetworkController

using System;

using System. Collections. Generic;

using System. Linq;

using System. Text;

using System. IO;

using System. Drawing;

using System. Threading;

using System. Diagnostics;

using grabberMolotok. DBEngine;

using grabberMolotok. Threading;

namespace FANNCategoriesDetector

{

class NetworkController

{

DBEngine dbEngine; // движок бд

List<NeuralNetwork> smallNetworks; // маленькие сети

List<NeuralNetwork> bigNetworks; // большие сети

public Dictionary<NeuralNetwork, double [] > workResult; // результат работы сетей

bool smallNetworksInitialized; // флаг инициализации малых сетей

public bool SmallNetworksInitialized

{

get { return smallNetworksInitialized; }

}

bool bigNetworksInitialized; // флаг инициализации больших сетей

public bool BigNetworksInitialized

{

get { return bigNetworksInitialized; }

set { bigNetworksInitialized = value; }

}

string initialDirectory; // рабочая директория

public string InitialDirectory

{

get { return initialDirectory; }

}

uint [] smallNetworkLayers = { 3072, 512, 1 }; // слои малых сетей

uint [] bigNetworkLayers = { 12000, 1000, 1 }; // слои больших сетей

int trainDataCreationCounter; // счетчик создания данных для обучения

int trainDataCreationWorkAmount; // объём работы данных обучения

int trainDataCreationThreadCount; // количество потоков данных обучения

public int TrainDataCreationThreadCount

{

get { return trainDataCreationThreadCount; }

set { trainDataCreationThreadCount = value; }

}

int networkTrainCounter; // счетчик обучения сетей

int networkTrainWorkAmount; // объём работы обучения сетей

int networkTrainThreadCount; // количество потоков обучения сетей

public int NetworkTrainThreadCount

{

get { return networkTrainThreadCount; }

set { networkTrainThreadCount = value; }

}

int networkCreateCounter; // счетчик создания сетей

int networkCreateWorkAmount; // объём создания сетей

int networkCreateThreadCount; // потоки создания сетей

public int NetworkCreateThreadCount

{

get { return networkCreateThreadCount; }

set { networkCreateThreadCount = value; }

}

#region События

public delegate void AllTrainDataCreationCompletedEventHandler ();

public static event AllTrainDataCreationCompletedEventHandler AllTrainDataCreationCompletedEvent;

public delegate void AllNetworkTrainedEventHandler ();

public static event AllNetworkTrainedEventHandler AllNetworkTrainedEvent;

public delegate void AllNetworkCreatedEventHandler ();

public static event AllNetworkCreatedEventHandler AllNetworkCreatedEvent;

public delegate void AllNetworkWorkedEventHandler ();

public static event AllNetworkWorkedEventHandler AllNetworkWorkedEvent;

public delegate void NetworkCreatedEventHandler ();

public static event NetworkCreatedEventHandler NetworkCreatedEvent;

public delegate void NetworkTrainedEventHandler ();

public static event NetworkTrainedEventHandler NetworkTrainedEvent;

void OnTrainDataCreationComplete ()

{

trainDataCreationCounter++;

if (trainDataCreationCounter == trainDataCreationWorkAmount)

{

if (AllTrainDataCreationCompletedEvent! = null)

AllTrainDataCreationCompletedEvent ();

}

}

void OnNetworkTrained ()

{

networkTrainCounter++;

if (networkTrainCounter == networkTrainWorkAmount)

{

if (AllNetworkTrainedEvent! = null)

AllNetworkTrainedEvent ();

}

}

void OnNetworkCreated ()

{

networkCreateCounter++;

if (networkCreateCounter == networkCreateWorkAmount)

{

if (AllNetworkCreatedEvent! = null)

AllNetworkCreatedEvent ();

smallNetworksInitialized = true;

}

}

#endregion События

public NetworkController ()

{

NetworkController.networkCreatedEvent += new NetworkController.networkCreatedEventHandler (this. OnNetworkCreated);

NetworkController.networkTrainedEvent += new NetworkController.networkTrainedEventHandler (this. OnNetworkTrained);

NeuralNetwork. TrainDataCreatedEvent += new NeuralNetwork. TrainDataCreatedEventHandler (this. OnTrainDataCreationComplete);

dbEngine = new grabberMolotok. DBEngine. DBEngine ();

smallNetworks = new List<NeuralNetwork> ();

initialDirectory = dbEngine. settingsTable. ReadSetting ("initialDirectory");

smallNetworksInitialized = false;

trainDataCreationCounter = 0;

networkTrainCounter = 0;

networkCreateCounter = 0;

trainDataCreationWorkAmount = - 1;

networkTrainWorkAmount = - 1;

networkTrainWorkAmount = - 1;

trainDataCreationThreadCount = 8;

networkTrainThreadCount = 8;

networkCreateThreadCount = 8;

}

public int GetNetworkCreationWorkAmount ()

{

var networks = (new DirectoryInfo (initialDirectory + @"\64x48")). GetFiles ("*. ann");

return networks. Length;

}

void CreateSmallNetworks ()

{

smallNetworks. Clear ();

Debug. WriteLine ("Инициализация нейронных сетей");

var networks = (new DirectoryInfo (initialDirectory + @"\64x48")). GetFiles ("*. ann");

networkCreateWorkAmount = networks. Length;

foreach (var n in networks)

{

Stopwatch watch = Stopwatch. StartNew ();

NeuralNetwork net = new NeuralNetwork (smallNetworkLayers);

net. Create (n. FullName);

Int32. TryParse (n. Name. Replace (". ann", ""), out net. id);

net. category = dbEngine. categoryTable. GetById (net. id);

smallNetworks. Add (net);

NetworkCreatedEvent ();

watch. Stop ();

Debug. WriteLine ( ( (double) watch. ElapsedTicks / (double) Stopwatch. Frequency). ToString ());

}

smallNetworksInitialized = true;

}

void CreateBigNetworks ()

{

bigNetworks. Clear ();

var networks = (new DirectoryInfo (initialDirectory + @"\400x300")). GetFiles ("*. ann");

foreach (var n in networks)

{

NeuralNetwork net = new NeuralNetwork (bigNetworkLayers);

net. Create (n. FullName);

Int32. TryParse (n. Name. Replace (". ann", ""), out net. id);

net. category = dbEngine. categoryTable. GetById (net. id);

bigNetworks. Add (net);

}

smallNetworksInitialized = true;

}

#region Паралельная инициализация сетей

public void ParallelCreateSmallNetworks ()

{

smallNetworks. Clear ();

var networksFilesArray = (new DirectoryInfo (initialDirectory + @"\64x48")). GetFiles ("*. ann");

List<FileInfo> networksFiles = new List<FileInfo> ();

networksFiles. AddRange (networksFilesArray);

networkCreateWorkAmount = networksFiles. Count;

var preparedList = ThreadHelper. SplitList (networksFiles, networkCreateThreadCount);

for (int counter = 0; counter < preparedList. Length; counter++)

{

ParameterForDelegate parameter = new ParameterForDelegate (preparedList, counter);

Thread thread = new Thread (new ParameterizedThreadStart (ParallelCreate));

thread. Start (parameter);

thread. Join (10);

}

}

class ParameterForDelegate

{

public List<FileInfo> [] preparedList;

public int counter;

public ParameterForDelegate (List<FileInfo> [] preparedList, int counter)

{

this. preparedList = preparedList;

this. counter = counter;

}

}

void ParallelCreate (object parameter)

{

List<FileInfo> [] preparedList = ( (ParameterForDelegate) parameter). preparedList;

int counter = ( (ParameterForDelegate) parameter). counter;

for (int j = 0; j < preparedList [counter]. Count; j++)

{

NeuralNetwork net = new NeuralNetwork (smallNetworkLayers);

net. Create (preparedList [counter] [j]. FullName);

Int32. TryParse (preparedList [counter] [j]. Name. Replace (". ann", ""), out net. id);

smallNetworks. Add (net);

Thread. Sleep (10);

if (NetworkCreatedEvent! = null)

NetworkCreatedEvent ();

}

}

#endregion Паралельная инициализация сетей

public /*Dictionary<NeuralNetwork, double [] >*/void Work (object image)

{

Bitmap imageToDetect = (Bitmap) image;

Dictionary<NeuralNetwork, double [] > result = new Dictionary<NeuralNetwork, double [] > ();

if (! smallNetworksInitialized)

CreateSmallNetworks ();

Debug. WriteLine ("Работа нейронных сетей");

foreach (var network in smallNetworks)

{

Stopwatch watch = Stopwatch. StartNew ();

result. Add (network, network. Work (imageToDetect));

watch. Stop ();

Debug. WriteLine ( ( (double) watch. ElapsedTicks / (double) Stopwatch. Frequency). ToString ());

}

workResult = result;

Thread. Sleep (50);

// return result;

AllNetworkWorkedEvent ();

}

public int GetTrainDataCreationWorkAmount ()

{

DirectoryInfo [] trainDirectoryInfo = (new DirectoryInfo (initialDirectory + @"\64x48")). GetDirectories ();

// return trainDirectoryInfo. Length * 2;

return trainDirectoryInfo. Length;

}

public int GetParallelTrainDataCreationWorkAmount ()

{

DirectoryInfo [] trainDirectoryInfo = (new DirectoryInfo (initialDirectory + @"\64x48")). GetDirectories ();

return trainDirectoryInfo. Length;

}

public void CreateSmallTrainData ()

{

WipeTrainData ();

DirectoryInfo [] trainDirectoryInfo = (new DirectoryInfo (initialDirectory + @"\64x48")). GetDirectories (); // Чтение каталога всех категорий

trainDataCreationWorkAmount = trainDirectoryInfo. Length;

for (int i = 0; i < trainDirectoryInfo. Length; i++)

{

DirectoryInfo [] items = trainDirectoryInfo [i]. GetDirectories (); // список товаров в категории

List<FileInfo> images = new List<FileInfo> (); // изображения товаров в категории

List<FileInfo> falseImages = new List<FileInfo> (); // изображения товаров в других категориях

foreach (var item in items)

{

images. AddRange (item. GetFiles ("*. jpg"));

}

for (int j = 0; j < trainDirectoryInfo. Length; j++)

{

if (j == i)

continue;

DirectoryInfo [] falseItems = trainDirectoryInfo [j]. GetDirectories ();

foreach (var item in falseItems)

{

falseImages. AddRange (item. GetFiles ("*. jpg"));

}

}

uint trainImagesCount = (uint) (images. Count + falseImages. Count);

NeuralNetwork net = new NeuralNetwork (smallNetworkLayers);

net. id = Convert. ToInt32 (trainDirectoryInfo [i]. Name);

net. CreateTrainDataFile (trainDirectoryInfo [i]. FullName, images, true);

// net. CreateTrainDataFile (trainDirectoryInfo [i]. FullName, images, true, trainImagesCount);

// net. CreateTrainDataFile (trainDirectoryInfo [i]. FullName, falseImages, false, trainImagesCount);

}

}

public void CreateSmallTrainData (List<DirectoryInfo> trainDirectoryInfo, List<DirectoryInfo> wholeTDI)

{

for (int i = 0; i < trainDirectoryInfo. Count; i++)

{

DirectoryInfo [] items = trainDirectoryInfo [i]. GetDirectories ();

List<FileInfo> images = new List<FileInfo> ();

List<FileInfo> falseImages = new List<FileInfo> ();

foreach (var item in items)

{

images. AddRange (item. GetFiles ("*. jpg"));

}

for (int j = 0; j < wholeTDI. Count; j++)

{

if (wholeTDI [j] == trainDirectoryInfo [i])

continue;

DirectoryInfo [] falseItems = wholeTDI [j]. GetDirectories ();

foreach (var item in falseItems)

{

falseImages. AddRange (item. GetFiles ("*. jpg"));

}

}

uint trainImagesCount = (uint) (images. Count + falseImages. Count);

NeuralNetwork net = new NeuralNetwork (smallNetworkLayers);

net. id = Convert. ToInt32 (trainDirectoryInfo [i]. Name);

// net. CreateTrainDataFile (trainDirectoryInfo [i]. FullName, images, true);

net. CreateTrainDataFile (trainDirectoryInfo [i]. FullName, images, true, trainImagesCount);

net. CreateTrainDataFile (trainDirectoryInfo [i]. FullName, falseImages, false, trainImagesCount);

}

}

public void CreateSmallTrainData (List<DirectoryInfo> trainDirectoryInfo)

{

for (int i = 0; i < trainDirectoryInfo. Count; i++)

{

DirectoryInfo [] items = trainDirectoryInfo [i]. GetDirectories ();

List<FileInfo> images = new List<FileInfo> ();

foreach (var item in items)

{

images. AddRange (item. GetFiles ("*. jpg"));

}

NeuralNetwork net = new NeuralNetwork (smallNetworkLayers);

net. id = Convert. ToInt32 (trainDirectoryInfo [i]. Name);

Stopwatch watch = Stopwatch. StartNew ();

net. CreateTrainDataFile (trainDirectoryInfo [i]. FullName, images, true);

watch. Stop ();

Debug. WriteLine ( ( (double) watch. ElapsedTicks / (double) Stopwatch. Frequency / (double) images. Count). ToString ());

}

}

public void ParallelCreateSmallTrainData ()

{

WipeTrainData ();

DirectoryInfo [] trainDirectoryInfo = (new DirectoryInfo (initialDirectory + @"\64x48")). GetDirectories (); // Чтение каталога всех категорий

trainDataCreationWorkAmount = trainDirectoryInfo. Length;

var preparedList = ThreadHelper. SplitList (trainDirectoryInfo. ToList<DirectoryInfo> (), trainDataCreationThreadCount);

foreach (List<DirectoryInfo> categories in preparedList)

{

Thread thread = new Thread (

delegate ()

{

Debug. WriteLine ("Создание файлов обучения");

CreateSmallTrainData (categories);

}

);

thread. Start ();

thread. Join (10);

}

}

public int GetNetworkTrainWorkAmount ()

{

DirectoryInfo [] trainDirectoryInfo = (new DirectoryInfo (initialDirectory + @"\64x48")). GetDirectories (); // Чтение каталога всех категорий

int counter = 0;

foreach (var category in trainDirectoryInfo)

{

string trainFilePath = category. FullName + " // " + category. Name +". tr";

if (File. Exists (trainFilePath))

{

counter++;

}

}

return counter;

}

public void TrainSmallNetworks ()

{

WipeNetworks ();

networkTrainWorkAmount = GetNetworkTrainWorkAmount ();

DirectoryInfo [] trainDirectoryInfo = (new DirectoryInfo (initialDirectory + @"\64x48")). GetDirectories (); // Чтение каталога всех категорий

foreach (var category in trainDirectoryInfo)

{

string trainFilePath = category. FullName + "\\" + category. Name +". tr";

if (File. Exists (trainFilePath))

{

NeuralNetwork net = new NeuralNetwork (smallNetworkLayers);

net. id = Convert. ToInt32 (category. Name);

Stopwatch watch = Stopwatch. StartNew ();

net. Train (trainFilePath);

watch. Stop ();

Debug. WriteLine ("Training " + ( (double) watch. ElapsedTicks / (double) Stopwatch. Frequency). ToString ());

Stopwatch anotherWatch = Stopwatch. StartNew ();

net. Save (initialDirectory + @"\64x48\");

anotherWatch. Stop ();

Debug. WriteLine ("Saving " + ( (double) anotherWatch. ElapsedTicks / (double) Stopwatch. Frequency). ToString ());

NetworkTrainedEvent ();

}

}

}

public void TrainSmallNetworks (List<DirectoryInfo> trainDirectoryInfo)

{

foreach (var category in trainDirectoryInfo)

{

string trainFilePath = category. FullName + " // " + category. Name +". tr";

if (File. Exists (trainFilePath))

{

NeuralNetwork net = new NeuralNetwork (smallNetworkLayers);

net. id = Convert. ToInt32 (category. Name);

// Stopwatch watch = Stopwatch. StartNew ();

net. Train (trainFilePath);

// watch. Stop ();

// Debug. WriteLine ( ( (double) watch. ElapsedTicks / (double) Stopwatch. Frequency). ToString ());

net. Save (initialDirectory + @"\64x48\");

NetworkTrainedEvent ();

}

}

}

public void ParallelTrainSmallNetworks ()

{

WipeNetworks ();

DirectoryInfo [] trainDirectoryInfo = (new DirectoryInfo (initialDirectory + @"\64x48")). GetDirectories (); // Чтение каталога всех категорий

networkTrainWorkAmount = GetNetworkTrainWorkAmount ();

var preparedList = ThreadHelper. SplitList (trainDirectoryInfo. ToList<DirectoryInfo> (), networkTrainThreadCount);

foreach (List<DirectoryInfo> categories in preparedList)

{

Thread thread = new Thread (

delegate ()

{

Debug. WriteLine ("Обучение сетей");

TrainSmallNetworks (categories);

}

);

thread. SetApartmentState (ApartmentState. STA);

thread. Start ();

thread. Join (50);

}

}

public void CascadeTrainSmallNetworks ()

{

DirectoryInfo [] trainDirectoryInfo = (new DirectoryInfo (initialDirectory + @"\64x48")). GetDirectories (); // Чтение каталога всех категорий

foreach (var category in trainDirectoryInfo)

{

string trainFilePath = category. FullName + " // " + category. Name +". tr";

NeuralNetwork net = new NeuralNetwork (smallNetworkLayers);

net. id = Convert. ToInt32 (category. Name);

net. CascadeTrain (trainFilePath);

net. Save (initialDirectory + @"\64x48\");

}

}

public void WipeTrainData ()

{

DirectoryInfo [] trainDirectoryInfo = (new DirectoryInfo (initialDirectory + @"\64x48")). GetDirectories ();

foreach (var dir in trainDirectoryInfo)

{

var tr = dir. GetFiles ("*. tr");

try

{

tr [0]. Delete ();

}

catch (Exception)

{

continue;

}

}

}

public void WipeNetworks ()

{

var ann = (new DirectoryInfo (initialDirectory + @"\64x48")). GetFiles ("*. ann");

foreach (var n in ann)

{

try

{

n. Delete ();

}

catch (Exception)

{

continue;

}

}

}

public void Wipe ()

{

WipeTrainData ();

WipeNetworks ();

}

}

}

Размещено на Allbest.ru